From 2a540206a74af8d38a01aaa5a37adc1008cad6ca Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Tue, 19 Aug 2014 13:57:00 -0300 Subject: [PATCH 0001/1853] fix xmliter namespace on selected node --- scrapy/utils/iterators.py | 22 +++++++++++++++------- tests/test_utils_iterators.py | 6 +++++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 150b077ae..11b873f2e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -20,19 +20,27 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) + DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S) HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S) + NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S) text = _body_or_str(obj) - header_start = re.search(HEADER_START_RE, text) - header_start = header_start.group(1).strip() if header_start else '' - header_end = re_rsearch(HEADER_END_RE, text) - header_end = text[header_end[1]:].strip() if header_end else '' + document_header = re.search(DOCUMENT_HEADER_RE, text) + document_header = document_header.group().strip() if document_header else '' + header_end_idx = re_rsearch(HEADER_END_RE, text) + header_end = text[header_end_idx[1]:].strip() if header_end_idx else '' + namespaces = {} + if header_end: + for tagname in reversed(re.findall(END_TAG_RE, header_end)): + tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + if tag: + namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): - nodetext = header_start + match.group() + header_end - yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] + nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + yield Selector(text=nodetext, type='xml') def csviter(obj, delimiter=None, headers=None, encoding=None): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fe53f831f..8b5941605 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -61,7 +61,6 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'item') - node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) @@ -74,6 +73,11 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').extract(), []) self.assertEqual(node.xpath('price/text()').extract(), []) + my_iter = self.xmliter(response, 'g:image_link') + node = next(my_iter) + node.register_namespace('g', 'http://base.google.com/ns/1.0') + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_exception(self): body = u"""onetwo""" From 706ed0e049fe008cac12f243371b67ee0230a08a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 14 Jun 2016 12:34:07 -0300 Subject: [PATCH 0002/1853] Spider middleware: process_spider_exception on generators --- docs/topics/exceptions.rst | 11 + docs/topics/spider-middleware.rst | 10 +- scrapy/core/spidermw.py | 37 +++- scrapy/exceptions.py | 5 + tests/test_spidermiddleware.py | 340 ++++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+), 14 deletions(-) create mode 100644 tests/test_spidermiddleware.py diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index cc02369d4..a3ff72827 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -62,6 +62,17 @@ remain disabled. Those components include: The exception must be raised in the component's ``__init__`` method. +InvalidOutput +------------- + +.. exception:: InvalidOutput + +This exception can be raised by a downloader or spider middleware to +indicate that some method returned a value not suported by the processing +chain. +See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware` +for a list of supported output values. + NotSupported ------------ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8360827e8..fc7669437 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -112,11 +112,12 @@ following methods: .. method:: process_spider_exception(response, exception, spider) - This method is called when a spider or :meth:`process_spider_input` - method (from other spider middleware) raises an exception. + This method is called when when a spider or :meth:`process_spider_input`/ + :meth:`process_spider_output` method (from other spider middleware) + raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Response`, dict or + iterable of :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. If it returns ``None``, Scrapy will continue processing this exception, @@ -125,7 +126,8 @@ following methods: exception reaches the engine (where it's logged and discarded). If it returns an iterable the :meth:`process_spider_output` pipeline - kicks in, and no other :meth:`process_spider_exception` will be called. + kicks in, starting with the last non-executed method, and no other + :meth:`process_spider_exception` will be called. :param response: the response being processed when the exception was raised diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a206e4b0c..0f03a7b36 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spider-middleware.rst """ import six from twisted.python.failure import Failure +from scrapy.exceptions import InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list @@ -40,31 +41,47 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) - assert result is None, \ - 'Middleware %s must returns None or ' \ - 'raise an exception, got %s ' \ - % (fname(method), type(result)) + if result is not None: + raise InvalidOutput('Middleware {} must return None or raise ' \ + 'an exception, got {}'.format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) def process_spider_exception(_failure): exception = _failure.value + # don't handle InvalidOutput exception + if isinstance(exception, InvalidOutput): + return _failure for method in self.methods['process_spider_exception']: result = method(response=response, exception=exception, spider=spider) - assert result is None or _isiterable(result), \ - 'Middleware %s must returns None, or an iterable object, got %s ' % \ - (fname(method), type(result)) + if result is not None and not _isiterable(result): + raise InvalidOutput('Middleware {} must return None or an iterable ' \ + 'object, got {}'.format(fname(method), type(result))) + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned if result is not None: return result return _failure def process_spider_output(result): + def wrapper(result_iterable): + try: + for r in result_iterable: + yield r + except Exception as ex: + exception_result = process_spider_exception(Failure(ex)) + if exception_result is None or isinstance(exception_result, Failure): + raise + for output in exception_result: + yield output for method in self.methods['process_spider_output']: result = method(response=response, result=result, spider=spider) - assert _isiterable(result), \ - 'Middleware %s must returns an iterable object, got %s ' % \ - (fname(method), type(result)) + if _isiterable(result): + result = wrapper(result) + else: + raise InvalidOutput('Middleware {} must return an iterable object, ' \ + 'got {}'.format(fname(method), type(result))) return result dfd = mustbe_deferred(process_spider_input, response) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 4bcecd994..ba7272255 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -11,6 +11,11 @@ class NotConfigured(Exception): """Indicates a missing configuration situation""" pass +class InvalidOutput(TypeError): + """Indicates an invalid value has been returned + by a middleware's processing method""" + pass + # HTTP and crawling class IgnoreRequest(Exception): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py new file mode 100644 index 000000000..8ec9583d8 --- /dev/null +++ b/tests/test_spidermiddleware.py @@ -0,0 +1,340 @@ + +import logging + +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase +from twisted.internet import defer + +from scrapy.spiders import Spider +from scrapy.item import Item, Field +from scrapy.http import Request +from scrapy.utils.test import get_crawler + + +class TestItem(Item): + value = Field() + + +# ================================================================================ +# exceptions from a spider's parse method +class BaseExceptionFromParseMethodSpider(Spider): + start_urls = ["http://example.com/"] + custom_settings = { + 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540} + } + + +class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider): + """ return value is NOT a generator """ + name = 'not_a_generator' + + def parse(self, response): + raise AssertionError + + +class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider): + """ return value is a generator; the exception is raised + before the items are yielded: no items should be scraped """ + name = 'generator_error_before_items' + + def parse(self, response): + raise ValueError + for i in range(3): + yield {'value': i} + + +class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider): + """ return value is a generator; the exception is raised + after the items are yielded: 3 items should be scraped """ + name = 'generator_error_after_items' + + def parse(self, response): + for i in range(3): + yield {'value': i} + raise FloatingPointError + + +class CatchExceptionMiddleware(object): + def process_spider_exception(self, response, exception, spider): + """ catch an exception and log it """ + logging.warn('{} exception caught'.format(exception.__class__.__name__)) + return None + + +# ================================================================================ +# exception from a previous middleware's process_spider_input method +# process_spider_input is not expected to return an iterable, so there are no +# separate tests for generator/non-generator implementations +class FromPreviousMiddlewareInputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'not_a_generator_from_previous_middleware_input' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return None + + +class RaiseExceptionOnInputMiddleware(object): + def process_spider_input(self, response, spider): + raise LookupError + + +# ================================================================================ +# exception from a previous middleware's process_spider_output method (not a generator) +class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'not_a_generator_from_previous_middleware_output' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return [{'value': i} for i in range(3)] + + +class RaiseExceptionOnOutputNotAGeneratorMiddleware(object): + def process_spider_output(self, response, result, spider): + raise UnicodeError + + +# ================================================================================ +# exception from a previous middleware's process_spider_output method (generator) +class GeneratorFromPreviousMiddlewareOutputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'generator_from_previous_middleware_output' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return [{'value': i} for i in range(10, 13)] + + +class RaiseExceptionOnOutputGeneratorMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + yield r + raise NameError + + +# ================================================================================ +# do something useful from the exception handler +class DoSomethingSpider(Spider): + start_urls = ["http://example.com"] + name = 'do_something' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.DoSomethingMiddleware': 540, + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, + # spider side + } + } + + def parse(self, response): + yield {'value': response.url} + raise ImportError + + +class DoSomethingMiddleware(object): + def process_spider_exception(self, response, exception, spider): + return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')] + + +# ================================================================================ +# don't catch InvalidOutput from scrapy's spider middleware manager +class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'invalid_return_value_from_previous_middleware_input' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540, + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return None + + +class InvalidReturnValueInputMiddleware(object): + def process_spider_input(self, response, spider): + return 1.0 # , not None + + +class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'invalid_return_value_from_previous_middleware_output' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return None + + +class InvalidReturnValueOutputMiddleware(object): + def process_spider_output(self, response, result, spider): + return 1 # , not an iterable + + +# ================================================================================ +# make sure only non already called process_spider_output methods +# are called if process_spider_exception returns an iterable +class ExecutionChainSpider(Spider): + start_urls = ["http://example.com"] + name = 'execution_chain' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.ThirdMiddleware': 540, + 'tests.test_spidermiddleware.SecondMiddleware': 541, + 'tests.test_spidermiddleware.FirstMiddleware': 542 + # spider side + }, + } + + def parse(self, response): + return None + + +class FirstMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + if isinstance(r, dict): + r['handled_by_first_middleware'] = True + yield r + + def process_spider_exception(self, response, exception, spider): + # log exception, handle control to the next middleware's process_spider_exception + logging.warn('{} exception caught'.format(exception.__class__.__name__)) + return None + + +class SecondMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + if isinstance(r, dict): + r['handled_by_second_middleware'] = True + yield r + raise MemoryError + + +class ThirdMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + if isinstance(r, dict): + r['handled_by_third_middleware'] = True + yield r + + def process_spider_exception(self, response, exception, spider): + # handle control to the next middleware's process_spider_output + return [{'item': i} for i in range(3)] + + +class TestSpiderMiddleware(TestCase): + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method(self): + # non-generator return value + crawler = get_crawler(NotAGeneratorSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("AssertionError exception caught", str(log)) + self.assertIn("spider_exceptions/AssertionError", str(log)) + # generator return value, no items before the error + crawler = get_crawler(GeneratorErrorBeforeItemsSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("ValueError exception caught", str(log)) + self.assertIn("spider_exceptions/ValueError", str(log)) + # generator return value, 3 items before the error + crawler = get_crawler(GeneratorErrorAfterItemsSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("'item_scraped_count': 3", str(log)) + self.assertIn("FloatingPointError exception caught", str(log)) + self.assertIn("spider_exceptions/FloatingPointError", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_previous_middleware_input(self): + crawler = get_crawler(FromPreviousMiddlewareInputSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("LookupError exception caught", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_previous_middleware_output(self): + # non-generator output value + crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertNotIn("UnicodeError exception caught", str(log)) + # generator output value + crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("'item_scraped_count': 3", str(log)) + self.assertIn("NameError exception caught", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_do_something(self): + crawler = get_crawler(DoSomethingSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("ImportError exception caught", str(log)) + self.assertIn("{'value': 10}", str(log)) + self.assertIn("{'value': 'asdf'}", str(log)) + self.assertIn("{'value': 'http://example.com'}", str(log)) + self.assertIn("{'value': 'http://example.org'}", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_invalid_return_value_previous_middleware(self): + """ don't catch InvalidOutput from middleware """ + # on middleware's input + crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) + with LogCapture() as log1: + yield crawler1.crawl() + self.assertNotIn("InvalidOutput exception caught", str(log1)) + self.assertIn("'spider_exceptions/InvalidOutput'", str(log1)) + # on middleware's output + crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) + with LogCapture() as log2: + yield crawler2.crawl() + self.assertNotIn("InvalidOutput exception caught", str(log2)) + self.assertIn("'spider_exceptions/InvalidOutput'", str(log2)) + + @defer.inlineCallbacks + def test_process_spider_exception_execution_chain(self): + # on middleware's input + crawler1 = get_crawler(ExecutionChainSpider) + with LogCapture() as log1: + yield crawler1.crawl() + self.assertNotIn("handled_by_first_middleware", str(log1)) + self.assertNotIn("handled_by_second_middleware", str(log1)) + self.assertIn("MemoryError exception caught", str(log1)) + self.assertIn("handled_by_third_middleware", str(log1)) From 4090cc3990636337964a6e157679d6be15ba6f3a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 8 Mar 2017 18:11:20 -0300 Subject: [PATCH 0003/1853] Spider middleware: use Mockserver to test process_spider_exception --- tests/test_spidermiddleware.py | 42 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 8ec9583d8..3981a8d75 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -9,16 +9,20 @@ from scrapy.spiders import Spider from scrapy.item import Item, Field from scrapy.http import Request from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer class TestItem(Item): value = Field() +class LocalhostSpider(Spider): + start_urls = ['http://localhost:8998'] # tests.mockserver.MockServer + + # ================================================================================ # exceptions from a spider's parse method -class BaseExceptionFromParseMethodSpider(Spider): - start_urls = ["http://example.com/"] +class BaseExceptionFromParseMethodSpider(LocalhostSpider): custom_settings = { 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540} } @@ -65,8 +69,7 @@ class CatchExceptionMiddleware(object): # exception from a previous middleware's process_spider_input method # process_spider_input is not expected to return an iterable, so there are no # separate tests for generator/non-generator implementations -class FromPreviousMiddlewareInputSpider(Spider): - start_urls = ["http://example.com/"] +class FromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'not_a_generator_from_previous_middleware_input' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -88,8 +91,7 @@ class RaiseExceptionOnInputMiddleware(object): # ================================================================================ # exception from a previous middleware's process_spider_output method (not a generator) -class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider): - start_urls = ["http://example.com/"] +class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): name = 'not_a_generator_from_previous_middleware_output' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -111,8 +113,7 @@ class RaiseExceptionOnOutputNotAGeneratorMiddleware(object): # ================================================================================ # exception from a previous middleware's process_spider_output method (generator) -class GeneratorFromPreviousMiddlewareOutputSpider(Spider): - start_urls = ["http://example.com/"] +class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): name = 'generator_from_previous_middleware_output' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -136,8 +137,7 @@ class RaiseExceptionOnOutputGeneratorMiddleware(object): # ================================================================================ # do something useful from the exception handler -class DoSomethingSpider(Spider): - start_urls = ["http://example.com"] +class DoSomethingSpider(LocalhostSpider): name = 'do_something' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -155,13 +155,12 @@ class DoSomethingSpider(Spider): class DoSomethingMiddleware(object): def process_spider_exception(self, response, exception, spider): - return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')] + return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')] # ================================================================================ # don't catch InvalidOutput from scrapy's spider middleware manager -class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider): - start_urls = ["http://example.com/"] +class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_input' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -181,8 +180,7 @@ class InvalidReturnValueInputMiddleware(object): return 1.0 # , not None -class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider): - start_urls = ["http://example.com/"] +class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_output' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -205,8 +203,7 @@ class InvalidReturnValueOutputMiddleware(object): # ================================================================================ # make sure only non already called process_spider_output methods # are called if process_spider_exception returns an iterable -class ExecutionChainSpider(Spider): - start_urls = ["http://example.com"] +class ExecutionChainSpider(LocalhostSpider): name = 'execution_chain' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -258,6 +255,13 @@ class ThirdMiddleware(object): class TestSpiderMiddleware(TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks def test_process_spider_exception_from_parse_method(self): # non-generator return value @@ -309,8 +313,8 @@ class TestSpiderMiddleware(TestCase): self.assertIn("ImportError exception caught", str(log)) self.assertIn("{'value': 10}", str(log)) self.assertIn("{'value': 'asdf'}", str(log)) - self.assertIn("{'value': 'http://example.com'}", str(log)) - self.assertIn("{'value': 'http://example.org'}", str(log)) + self.assertIn("{'value': 'http://localhost:8998'}", str(log)) + self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log)) @defer.inlineCallbacks def test_process_spider_exception_invalid_return_value_previous_middleware(self): From 9c256cf693d73e854d409d717854b3f354b5e0a9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Mar 2017 15:41:57 -0300 Subject: [PATCH 0004/1853] Undocument _InvalidOutput exception --- docs/topics/exceptions.rst | 11 ----------- scrapy/core/spidermw.py | 12 ++++++------ scrapy/exceptions.py | 8 +++++--- tests/test_spidermiddleware.py | 12 ++++++------ 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index a3ff72827..cc02369d4 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -62,17 +62,6 @@ remain disabled. Those components include: The exception must be raised in the component's ``__init__`` method. -InvalidOutput -------------- - -.. exception:: InvalidOutput - -This exception can be raised by a downloader or spider middleware to -indicate that some method returned a value not suported by the processing -chain. -See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware` -for a list of supported output values. - NotSupported ------------ diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0f03a7b36..50677670b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,7 +5,7 @@ See documentation in docs/topics/spider-middleware.rst """ import six from twisted.python.failure import Failure -from scrapy.exceptions import InvalidOutput +from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list @@ -42,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - raise InvalidOutput('Middleware {} must return None or raise ' \ + raise _InvalidOutput('Middleware {} must return None or raise ' \ 'an exception, got {}'.format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) @@ -50,13 +50,13 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_spider_exception(_failure): exception = _failure.value - # don't handle InvalidOutput exception - if isinstance(exception, InvalidOutput): + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): return _failure for method in self.methods['process_spider_exception']: result = method(response=response, exception=exception, spider=spider) if result is not None and not _isiterable(result): - raise InvalidOutput('Middleware {} must return None or an iterable ' \ + raise _InvalidOutput('Middleware {} must return None or an iterable ' \ 'object, got {}'.format(fname(method), type(result))) # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned @@ -80,7 +80,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = wrapper(result) else: - raise InvalidOutput('Middleware {} must return an iterable object, ' \ + raise _InvalidOutput('Middleware {} must return an iterable object, ' \ 'got {}'.format(fname(method), type(result))) return result diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index ba7272255..96949bdd9 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -11,9 +11,11 @@ class NotConfigured(Exception): """Indicates a missing configuration situation""" pass -class InvalidOutput(TypeError): - """Indicates an invalid value has been returned - by a middleware's processing method""" +class _InvalidOutput(TypeError): + """ + Indicates an invalid value has been returned by a middleware's processing method. + Internal and undocumented, it should not be raised or caught by user code. + """ pass # HTTP and crawling diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 3981a8d75..2d05c335c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -159,7 +159,7 @@ class DoSomethingMiddleware(object): # ================================================================================ -# don't catch InvalidOutput from scrapy's spider middleware manager +# don't catch _InvalidOutput from scrapy's spider middleware manager class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_input' custom_settings = { @@ -318,19 +318,19 @@ class TestSpiderMiddleware(TestCase): @defer.inlineCallbacks def test_process_spider_exception_invalid_return_value_previous_middleware(self): - """ don't catch InvalidOutput from middleware """ + """ don't catch _InvalidOutput from middleware """ # on middleware's input crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) with LogCapture() as log1: yield crawler1.crawl() - self.assertNotIn("InvalidOutput exception caught", str(log1)) - self.assertIn("'spider_exceptions/InvalidOutput'", str(log1)) + self.assertNotIn("_InvalidOutput exception caught", str(log1)) + self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) # on middleware's output crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) with LogCapture() as log2: yield crawler2.crawl() - self.assertNotIn("InvalidOutput exception caught", str(log2)) - self.assertIn("'spider_exceptions/InvalidOutput'", str(log2)) + self.assertNotIn("_InvalidOutput exception caught", str(log2)) + self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) @defer.inlineCallbacks def test_process_spider_exception_execution_chain(self): From 4cfbe8204480214b65d48caaf080feda30fe91ae Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Mar 2017 15:47:47 -0300 Subject: [PATCH 0005/1853] Downloader middleware: raise _InvalidOutput Instead of AssertionError, to make it consistent with spider middleware --- scrapy/core/downloader/middleware.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index c3b23e284..cf0c1f869 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -7,6 +7,7 @@ import six from twisted.internet import defer +from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred @@ -35,9 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_request(request): for method in self.methods['process_request']: response = yield method(request=request, spider=spider) - assert response is None or isinstance(response, (Response, Request)), \ - 'Middleware %s.process_request must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, response.__class__.__name__) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \ + (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) defer.returnValue((yield download_func(request=request,spider=spider))) @@ -51,9 +52,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_response']: response = yield method(request=request, response=response, spider=spider) - assert isinstance(response, (Response, Request)), \ - 'Middleware %s.process_response must return Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response)) + if not isinstance(response, (Response, Request)): + raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ + (six.get_method_self(method).__class__.__name__, type(response))) if isinstance(response, Request): defer.returnValue(response) defer.returnValue(response) @@ -64,9 +65,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_exception']: response = yield method(request=request, exception=exception, spider=spider) - assert response is None or isinstance(response, (Response, Request)), \ - 'Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response)) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ + (six.get_method_self(method).__class__.__name__, type(response))) if response: defer.returnValue(response) defer.returnValue(_failure) From b040df5ac09cf133cd07b505d20469a56409129c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 21 Mar 2017 15:56:18 +0500 Subject: [PATCH 0006/1853] TST cleanup spider middleware tests --- tests/test_spidermiddleware.py | 73 ++++++++++++++++------------------ 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2d05c335c..109bcc250 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -255,61 +255,62 @@ class ThirdMiddleware(object): class TestSpiderMiddleware(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method(self): - # non-generator return value - crawler = get_crawler(NotAGeneratorSpider) + def crawl_log(self, spider): + crawler = get_crawler(spider) with LogCapture() as log: yield crawler.crawl() + raise defer.returnValue(log) + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method_non_generator(self): + # non-generator return value + log = yield self.crawl_log(NotAGeneratorSpider) self.assertIn("AssertionError exception caught", str(log)) self.assertIn("spider_exceptions/AssertionError", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method_generator_no_items(self): # generator return value, no items before the error - crawler = get_crawler(GeneratorErrorBeforeItemsSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider) self.assertIn("ValueError exception caught", str(log)) self.assertIn("spider_exceptions/ValueError", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method_generator_with_items(self): # generator return value, 3 items before the error - crawler = get_crawler(GeneratorErrorAfterItemsSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(GeneratorErrorAfterItemsSpider) self.assertIn("'item_scraped_count': 3", str(log)) self.assertIn("FloatingPointError exception caught", str(log)) self.assertIn("spider_exceptions/FloatingPointError", str(log)) @defer.inlineCallbacks def test_process_spider_exception_from_previous_middleware_input(self): - crawler = get_crawler(FromPreviousMiddlewareInputSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(FromPreviousMiddlewareInputSpider) self.assertIn("LookupError exception caught", str(log)) @defer.inlineCallbacks def test_process_spider_exception_from_previous_middleware_output(self): # non-generator output value - crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider) self.assertNotIn("UnicodeError exception caught", str(log)) # generator output value - crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider) self.assertIn("'item_scraped_count': 3", str(log)) self.assertIn("NameError exception caught", str(log)) @defer.inlineCallbacks def test_process_spider_exception_do_something(self): - crawler = get_crawler(DoSomethingSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(DoSomethingSpider) self.assertIn("ImportError exception caught", str(log)) self.assertIn("{'value': 10}", str(log)) self.assertIn("{'value': 'asdf'}", str(log)) @@ -320,25 +321,19 @@ class TestSpiderMiddleware(TestCase): def test_process_spider_exception_invalid_return_value_previous_middleware(self): """ don't catch _InvalidOutput from middleware """ # on middleware's input - crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) - with LogCapture() as log1: - yield crawler1.crawl() + log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider) self.assertNotIn("_InvalidOutput exception caught", str(log1)) self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) # on middleware's output - crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) - with LogCapture() as log2: - yield crawler2.crawl() + log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider) self.assertNotIn("_InvalidOutput exception caught", str(log2)) self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) @defer.inlineCallbacks def test_process_spider_exception_execution_chain(self): # on middleware's input - crawler1 = get_crawler(ExecutionChainSpider) - with LogCapture() as log1: - yield crawler1.crawl() - self.assertNotIn("handled_by_first_middleware", str(log1)) - self.assertNotIn("handled_by_second_middleware", str(log1)) - self.assertIn("MemoryError exception caught", str(log1)) - self.assertIn("handled_by_third_middleware", str(log1)) + log = yield self.crawl_log(ExecutionChainSpider) + self.assertNotIn("handled_by_first_middleware", str(log)) + self.assertNotIn("handled_by_second_middleware", str(log)) + self.assertIn("MemoryError exception caught", str(log)) + self.assertIn("handled_by_third_middleware", str(log)) From 815d6160cfac5c0b764d01fc9d4ebc4ab7793aac Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 16 Nov 2015 18:28:49 +0100 Subject: [PATCH 0007/1853] Add from_crawler constructor for feed exporters and storages --- scrapy/extensions/feedexport.py | 12 +++++-- scrapy/middleware.py | 9 ++--- scrapy/utils/misc.py | 24 +++++++++++++ tests/test_feedexport.py | 30 ++++++++++++++++ tests/test_utils_misc/__init__.py | 58 ++++++++++++++++++++++++++++++- 5 files changed, 122 insertions(+), 11 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5f133fbde..70c302fba 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,7 +21,7 @@ from w3lib.url import file_uri_to_path from scrapy import signals from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured -from scrapy.utils.misc import load_object +from scrapy.utils.misc import create_instance, load_object from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import without_none_values from scrapy.utils.boto import is_botocore @@ -181,6 +181,7 @@ class FeedExporter(object): @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings) + o.crawler = crawler crawler.signals.connect(o.open_spider, signals.spider_opened) crawler.signals.connect(o.close_spider, signals.spider_closed) crawler.signals.connect(o.item_scraped, signals.item_scraped) @@ -253,11 +254,16 @@ class FeedExporter(object): logger.error("Unknown feed storage scheme: %(scheme)s", {'scheme': scheme}) + def _get_instance(self, objcls, *args, **kwargs): + return create_instance( + objcls, self.settings, getattr(self, 'crawler', None), + *args, **kwargs) + def _get_exporter(self, *args, **kwargs): - return self.exporters[self.format](*args, **kwargs) + return self._get_instance(self.exporters[self.format], *args, **kwargs) def _get_storage(self, uri): - return self.storages[urlparse(uri).scheme](uri) + return self._get_instance(self.storages[urlparse(uri).scheme], uri) def _get_uri_params(self, spider): params = {} diff --git a/scrapy/middleware.py b/scrapy/middleware.py index be36f977e..f2240984c 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,7 +3,7 @@ import logging import pprint from scrapy.exceptions import NotConfigured -from scrapy.utils.misc import load_object +from scrapy.utils.misc import create_instance, load_object from scrapy.utils.defer import process_parallel, process_chain, process_chain_both logger = logging.getLogger(__name__) @@ -32,12 +32,7 @@ class MiddlewareManager(object): for clspath in mwlist: try: mwcls = load_object(clspath) - if crawler and hasattr(mwcls, 'from_crawler'): - mw = mwcls.from_crawler(crawler) - elif hasattr(mwcls, 'from_settings'): - mw = mwcls.from_settings(settings) - else: - mw = mwcls() + mw = create_instance(mwcls, settings, crawler) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 35f855007..8eb1aabb5 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -117,3 +117,27 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False + +def create_instance(objcls, settings, crawler, *args, **kwargs): + """Construct a class instance using its ``from_crawler`` or + ``from_settings`` constructors, if available. + + At least one of ``settings`` and ``crawler`` needs to be different from + ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. + If ``crawler`` is ``None``, only the ``from_settings`` constructor will be + tried. + + ``*args`` and ``**kwargs`` are forwarded to the constructors. + + Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. + """ + if settings is None: + if crawler is None: + raise ValueError("Specifiy at least one of settings and crawler.") + settings = crawler.settings + if crawler and hasattr(objcls, 'from_crawler'): + return objcls.from_crawler(crawler, *args, **kwargs) + elif hasattr(objcls, 'from_settings'): + return objcls.from_settings(settings, *args, **kwargs) + else: + return objcls(*args, **kwargs) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f55927121..08f7e4d8d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -16,6 +16,7 @@ from tests.mockserver import MockServer from w3lib.url import path_to_file_uri import scrapy +from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage, @@ -159,6 +160,23 @@ class StdoutFeedStorageTest(unittest.TestCase): self.assertEqual(out.getvalue(), b"content") +class FromCrawlerMixin(object): + init_with_crawler = False + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + cls.init_with_crawler = True + return cls(*args, **kwargs) + + +class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin): + pass + + +class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): + pass + + class FeedExportTest(unittest.TestCase): class MyItem(scrapy.Item): @@ -599,3 +617,15 @@ class FeedExportTest(unittest.TestCase): data = yield self.exported_data(items, settings) print(row['format'], row['indent']) self.assertEqual(row['expected'], data) + + @defer.inlineCallbacks + def test_init_exporters_storages_with_crawler(self): + settings = { + 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.' + 'FromCrawlerCsvItemExporter'}, + 'FEED_STORAGES': {'file': 'tests.test_feedexport.' + 'FromCrawlerFileFeedStorage'}, + } + yield self.exported_data({}, settings) + self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler) + self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..b95bba5c1 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,9 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import load_object, arg_to_iter, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules + +from tests import mock __doctests__ = ['scrapy.utils.misc'] @@ -74,5 +76,59 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) + def test_create_instance(self): + settings = mock.MagicMock() + crawler = mock.MagicMock(spec_set=['settings']) + args = (True, 100.) + kwargs = {'key': 'val'} + + def _test_with_settings(mock, settings): + create_instance(mock, settings, None, *args, **kwargs) + if hasattr(mock, 'from_crawler'): + self.assertEqual(mock.from_crawler.call_count, 0) + if hasattr(mock, 'from_settings'): + mock.from_settings.assert_called_once_with(settings, *args, + **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + def _test_with_crawler(mock, settings, crawler): + create_instance(mock, settings, crawler, *args, **kwargs) + if hasattr(mock, 'from_crawler'): + mock.from_crawler.assert_called_once_with(crawler, *args, + **kwargs) + if hasattr(mock, 'from_settings'): + self.assertEqual(mock.from_settings.call_count, 0) + self.assertEqual(mock.call_count, 0) + elif hasattr(mock, 'from_settings'): + mock.from_settings.assert_called_once_with(settings, *args, + **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using four mocks: + # 1. with no alternative constructors + # 2. with from_settings() constructor + # 3. with from_crawler() constructor + # 4. with from_settings() and from_crawler() constructor + spec_sets = ([], ['from_settings'], ['from_crawler'], + ['from_settings', 'from_crawler']) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_settings(m, settings) + m.reset_mock() + _test_with_crawler(m, settings, crawler) + + # Check adoption of crawler settings + m = mock.MagicMock(spec_set=['from_settings']) + create_instance(m, None, crawler, *args, **kwargs) + m.from_settings.assert_called_once_with(crawler.settings, *args, + **kwargs) + + with self.assertRaises(ValueError): + create_instance(m, None, None) + if __name__ == "__main__": unittest.main() From 4d77c3084e64d351e3acf9092b4171d4418925fe Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 23 Nov 2015 12:24:05 +0100 Subject: [PATCH 0008/1853] Add from_crawler constructor to S3FeedStorage --- scrapy/extensions/feedexport.py | 27 ++++++++++++++++++++--- tests/test_feedexport.py | 38 ++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 70c302fba..067887d94 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -93,12 +93,28 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri): + def __init__(self, uri, access_key=None, secret_key=None): + # BEGIN Backwards compatibility for initialising without keys (and + # without using from_crawler) from scrapy.conf import settings + no_defaults = access_key is None and secret_key is None + if no_defaults and ('AWS_ACCESS_KEY_ID' in settings or + 'AWS_SECRET_ACCESS_KEY' in settings): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn( + "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " + "without AWS keys is deprecated. Please supply credentials or " + "use the `from_crawler()` constructor.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + access_key = settings['AWS_ACCESS_KEY_ID'] + secret_key = settings['AWS_SECRET_ACCESS_KEY'] u = urlparse(uri) self.bucketname = u.hostname - self.access_key = u.username or settings['AWS_ACCESS_KEY_ID'] - self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY'] + self.access_key = u.username or access_key + self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" if self.is_botocore: @@ -111,6 +127,11 @@ class S3FeedStorage(BlockingFeedStorage): import boto self.connect_s3 = boto.connect_s3 + @classmethod + def from_crawler(cls, crawler, uri): + return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], + crawler.settings['AWS_SECRET_ACCESS_KEY']) + def _store_in_thread(self, file): file.seek(0) if self.is_botocore: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08f7e4d8d..eeb1bc2a4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -6,12 +6,14 @@ from io import BytesIO import tempfile import shutil from six.moves.urllib.parse import urlparse +import warnings from zope.interface.verify import verifyObject from twisted.trial import unittest from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings +from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri @@ -131,13 +133,47 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): + @mock.patch('scrapy.conf.settings', new={'AWS_ACCESS_KEY_ID': 'conf_key', + 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}, create=True) + def test_parse_credentials(self): + try: + import boto + except ImportError: + raise unittest.SkipTest("S3FeedStorage requires boto") + aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', + 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} + crawler = get_crawler(settings_dict=aws_credentials) + # Instantiate with crawler + storage = S3FeedStorage.from_crawler(crawler, + 's3://mybucket/export.csv') + self.assertEqual(storage.access_key, 'settings_key') + self.assertEqual(storage.secret_key, 'settings_secret') + # Instantiate directly + storage = S3FeedStorage('s3://mybucket/export.csv', + aws_credentials['AWS_ACCESS_KEY_ID'], + aws_credentials['AWS_SECRET_ACCESS_KEY']) + self.assertEqual(storage.access_key, 'settings_key') + self.assertEqual(storage.secret_key, 'settings_secret') + # URI priority > settings priority + storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv', + aws_credentials['AWS_ACCESS_KEY_ID'], + aws_credentials['AWS_SECRET_ACCESS_KEY']) + self.assertEqual(storage.access_key, 'uri_key') + self.assertEqual(storage.secret_key, 'uri_secret') + # Backwards compatibility for initialising without settings + with warnings.catch_warnings(record=True) as w: + storage = S3FeedStorage('s3://mybucket/export.csv') + self.assertEqual(storage.access_key, 'conf_key') + self.assertEqual(storage.secret_key, 'conf_secret') + self.assertTrue('without AWS keys' in str(w[-1].message)) + @defer.inlineCallbacks def test_store(self): assert_aws_environ() uri = os.environ.get('S3_TEST_FILE_URI') if not uri: raise unittest.SkipTest("No S3 URI available for testing") - storage = S3FeedStorage(uri) + storage = S3FeedStorage(uri, Settings()) verifyObject(IFeedStorage, storage) file = storage.open(scrapy.Spider("default")) expected_content = b"content: \xe2\x98\x83" From aca2655c12d806759c6e0821a40d0277d200e0ea Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 14:57:39 +0100 Subject: [PATCH 0009/1853] [WIP] Run tests for Python 3.7 --- .travis.yml | 2 ++ tox.ini | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..e4df22139 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs + - python: 3.7 + env: TOXENV=py37 install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/tox.ini b/tox.ini index 60ff8c15e..5301624ee 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,10 @@ deps = {[testenv:py34]deps} basepython = python3.6 deps = {[testenv:py34]deps} +[testenv:py37] +basepython = python3.7 +deps = {[testenv:py34]deps} + [testenv:pypy3] basepython = pypy3 deps = {[testenv:py34]deps} From fab68ff6260b9ce4f55ca7b211a1aeb3e8e6df3d Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 17:05:14 +0100 Subject: [PATCH 0010/1853] Use 3.7-dev version for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e4df22139..aa1a3c4c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7 + - python: 3.7-dev env: TOXENV=py37 install: - | From 4c05441450bc1f8438239af0310ab76777a2dacf Mon Sep 17 00:00:00 2001 From: nctl144 Date: Sat, 3 Mar 2018 00:00:03 -0500 Subject: [PATCH 0011/1853] add ftp to the scheme list --- scrapy/linkextractors/__init__.py | 3 ++- tests/test_linkextractors.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..cda6ddc7e 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -41,7 +41,8 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \ + 'file', 'ftp'} class FilteringLinkExtractor(object): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 1d7c4f311..903032b52 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -451,6 +451,17 @@ class Base: Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_ftp_links(self): + body = b""" + + + """ + response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + ]) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor @@ -471,4 +482,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() - From d5b7ebcfdcfd40d29990712b587893fcc6e84ce8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 18:17:49 +0530 Subject: [PATCH 0012/1853] Fixed bug FormRequest.from_response() clickdata ignores input[type=image] --- scrapy/http/request/form.py | 10 ++++++---- tests/test_http_request.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d9d178a3e..184ee2599 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,10 +170,12 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: return diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fca8ff411..73a74cd5d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,6 +532,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( From 65744c2199fc6a5bccfa11eec40a867c1401aee9 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 20:07:50 +0530 Subject: [PATCH 0013/1853] Corrected Test --- tests/test_http_request.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 73a74cd5d..a042f03b6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,16 +532,6 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""") - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -554,6 +544,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_multiple_clickdata(self): response = _buildresponse( From ca7d79c29a55be4482bf2d4f704fc145ba801337 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 10:46:51 +0100 Subject: [PATCH 0014/1853] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..47eddf1fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +git+https://github.com/twisted/twisted.git@dcaf946 lxml pyOpenSSL cssselect>=0.9 From 5d1f5245f2699745e73449b013a29cc370f424c9 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:14:50 +0100 Subject: [PATCH 0015/1853] [WIP] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47eddf1fc..95cd37772 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/twisted/twisted.git@dcaf946 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml pyOpenSSL cssselect>=0.9 From f10a43d562dee32f324703fddb19bee5266912ce Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:43:39 +0100 Subject: [PATCH 0016/1853] [WIP] Install Twisted from branch for py3.7 --- requirements-py3.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..c3357e970 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/requirements.txt b/requirements.txt index 95cd37772..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 From 13a74d77e2e4d1719bf984d99ca2ae6874752e41 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 12 Mar 2018 22:25:19 +0800 Subject: [PATCH 0017/1853] catch CertificateError in tls verification --- scrapy/core/downloader/tls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 498e3d60f..e1c4f4908 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,6 +40,7 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) + from service_identity.exceptions import CertificateError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -65,7 +66,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except VerificationError as e: + except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) From e487100987496474de4d2b5fc509ff47d01cd0b6 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 08:59:03 +0800 Subject: [PATCH 0018/1853] add a test case --- tests/keys/localhost-ip.gen.README | 21 ++++++++++++ tests/keys/localhost.crt | 36 ++++++++++----------- tests/keys/localhost.ip.crt | 20 ++++++++++++ tests/keys/localhost.ip.key | 28 ++++++++++++++++ tests/keys/localhost.key | 52 +++++++++++++++--------------- tests/test_downloader_handlers.py | 9 ++++++ 6 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 tests/keys/localhost-ip.gen.README create mode 100644 tests/keys/localhost.ip.crt create mode 100644 tests/keys/localhost.ip.key diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README new file mode 100644 index 000000000..8e94e1217 --- /dev/null +++ b/tests/keys/localhost-ip.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1 +Email Address []:. + diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 13c5b5bd6..48d7bd9a3 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x -NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K -o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y -njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY -ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ -8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 -gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O -BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j -xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES -9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i -jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ -TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own -tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM -am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U -Mjybr/hQK3mnVdE= +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.crt b/tests/keys/localhost.ip.crt new file mode 100644 index 000000000..48d7bd9a3 --- /dev/null +++ b/tests/keys/localhost.ip.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.key b/tests/keys/localhost.ip.key new file mode 100644 index 000000000..1e12c1255 --- /dev/null +++ b/tests/keys/localhost.ip.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq +-----END PRIVATE KEY----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index da975e6d3..1e12c1255 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 -ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 -JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz -vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 -RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY -b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ -XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 -gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk -i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu -Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss -ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 -hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD -7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa -EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf -uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG -HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR -UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO -j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri -fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ -6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI -xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g -DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f -npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM -8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 -J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj -Z+pMCFLbwAbFZ9Qb/NFlUv0= +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq -----END PRIVATE KEY----- diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bd2c86292..ceb03f945 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -505,6 +505,15 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' +class Https11InvalidDNSPattern(Https11TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + super(Https11InvalidDNSPattern, self).setUp() + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" From d9e6c73fb3ef787e39474bda5f008b309b65c65b Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 13:05:37 +0800 Subject: [PATCH 0019/1853] revert wrong changes --- tests/keys/localhost.crt | 36 ++++++++++++++-------------- tests/keys/localhost.key | 52 ++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 48d7bd9a3..13c5b5bd6 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x -ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL -JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev -y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV -ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR -v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 -JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O -BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ -dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr -idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP -8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X -kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ -FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb -smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO -zh6AcsnjkLumOkM= +MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x +NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K +o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y +njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY +ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ +8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 +gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O +BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j +xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES +9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i +jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ +TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own +tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM +am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U +Mjybr/hQK3mnVdE= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index 1e12c1255..da975e6d3 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo -+g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC -T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb -rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR -3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi -JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj -uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES -FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc -BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud -JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD -bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT -ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx -KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N -yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA -hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ -32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie -PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY -r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 -X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz -ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ -Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ -g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 -WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ -inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u -BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz -m1Q+9xJdtRxlYc1YGTK1YGCq +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 +ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 +JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz +vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 +RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY +b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ +XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 +gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk +i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu +Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss +ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 +hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD +7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa +EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf +uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG +HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR +UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO +j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri +fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ +6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI +xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g +DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f +npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM +8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 +J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj +Z+pMCFLbwAbFZ9Qb/NFlUv0= -----END PRIVATE KEY----- From 6a7cdf9a6c4162cf4dd91721d28974c80e0f68cd Mon Sep 17 00:00:00 2001 From: siulkilulki Date: Tue, 13 Mar 2018 08:35:27 +0100 Subject: [PATCH 0020/1853] [MRG+1] Add 'flv' to ignored video extensions. (#3165) --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..c3c79cf25 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', 'm4v', + 'm4a', 'm4v', 'flv', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', From 1a2f0193a30c1aceb4743e33a90b3c264e0b09c4 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 19:14:52 +0800 Subject: [PATCH 0021/1853] fix tests on jessie --- scrapy/core/downloader/tls.py | 11 +++++++++-- tests/test_downloader_handlers.py | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e1c4f4908..c97c6a9a9 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,7 +40,14 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) - from service_identity.exceptions import CertificateError + try: + # XXX: this import would fail on Debian jessie with system installed + # service_identity library, due to lack of cryptography.x509 dependency + # See https://github.com/pyca/service_identity/issues/21 + from service_identity.exceptions import CertificateError + verification_errors = (CertificateError, VerificationError) + except ImportError: + verification_errors = VerificationError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -66,7 +73,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except (CertificateError, VerificationError) as e: + except verification_errors as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ceb03f945..b34faa7e7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -512,6 +512,10 @@ class Https11InvalidDNSPattern(Https11TestCase): certfile = 'keys/localhost.ip.crt' def setUp(self): + try: + from service_identity.exceptions import CertificateError + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") super(Https11InvalidDNSPattern, self).setUp() From 2c58da19a6f85f4532d80d3a941cede8f9d0bab8 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 14 Mar 2018 09:27:59 +0800 Subject: [PATCH 0022/1853] update docstring of ScrapyClientTLSOptions --- scrapy/core/downloader/tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index c97c6a9a9..df8051182 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -63,8 +63,9 @@ if twisted_version >= (14, 0, 0): (for genuinely invalid certificates or bugs in verification code). Same as Twisted's private _sslverify.ClientTLSOptions, - except that VerificationError and ValueError exceptions are caught, - so that the connection is not closed, only logging warnings. + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. """ def _identityVerifyingInfoCallback(self, connection, where, ret): From ff5f717f7a2aaf0a1a1101019485201427edd536 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:17:48 +0530 Subject: [PATCH 0023/1853] Fixed formatting issues --- scrapy/http/request/form.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 184ee2599..22846ad77 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,15 +168,17 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ + print("form =", form.__dict__) clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] + print("clickables =", clickables) if not clickables: return From e25e2afe174bbe70ecbf88ea684937890e8ba4d5 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:20:14 +0530 Subject: [PATCH 0024/1853] Removed unnecessary print statements --- scrapy/http/request/form.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 22846ad77..238dd44b3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,7 +168,6 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - print("form =", form.__dict__) clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' @@ -178,7 +177,6 @@ def _get_clickable(clickdata, form): '|descendant::button[not(@type)]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] - print("clickables =", clickables) if not clickables: return From a5acc9373f8735e27c22de6fbe345fbed8f268c1 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 18:19:39 +0530 Subject: [PATCH 0025/1853] Resolving Comments --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 238dd44b3..d033a830e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,11 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', + 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From dd064413a46356940151ac3f9ccd8a45bca2cbd8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 19:28:41 +0530 Subject: [PATCH 0026/1853] corrected syntax error in XPath --- scrapy/http/request/form.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d033a830e..95b38e990 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,8 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From c6d20bdd826070c6c808421cd6331b5d208f8aa7 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Tue, 27 Mar 2018 16:21:07 -0400 Subject: [PATCH 0027/1853] Doc: update wording for COOKIES_ENABLED --- docs/topics/downloader-middleware.rst | 20 +++++++++----------- docs/topics/request-response.rst | 4 +++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 863620900..dfe4c13b4 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -158,13 +158,13 @@ more of the following methods: :type spider: :class:`~scrapy.spiders.Spider` object .. method:: from_crawler(cls, crawler) - + If present, this classmethod is called to create a middleware instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the middleware. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for middleware to access them and hook its functionality into Scrapy. - + :param crawler: crawler that uses this middleware :type crawler: :class:`~scrapy.crawler.Crawler` object @@ -237,16 +237,14 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. -Notice that if the :class:`~scrapy.http.Request` -has ``meta['dont_merge_cookies']`` evaluated to ``True``. -despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be -sent to web servers and received cookies in -:class:`~scrapy.http.Response` will **not** be merged with the existing -cookies. - -For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request` +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. .. setting:: COOKIES_DEBUG diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 121abe6b5..e29914dbf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -80,6 +80,8 @@ Request objects attributes of the cookie. This is only useful if the cookies are saved for later requests. + .. reqmeta:: dont_merge_cookies + When some site returns cookies (in a response) those are stored in the cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. However, if, for some @@ -294,7 +296,7 @@ Those are: * :reqmeta:`dont_retry` * :reqmeta:`handle_httpstatus_list` * :reqmeta:`handle_httpstatus_all` -* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor) +* :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` * :reqmeta:`redirect_urls` From 8e8994c6b55fa8e975ce30b25ae326f829a58aed Mon Sep 17 00:00:00 2001 From: rhoboro Date: Mon, 2 Apr 2018 15:36:47 +0900 Subject: [PATCH 0028/1853] add acl support for gcs --- scrapy/pipelines/files.py | 6 +++++- scrapy/pipelines/images.py | 1 + scrapy/settings/default_settings.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ab18a727d..af1d5488a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -207,6 +207,8 @@ class GCSFilesStore(object): GCS_PROJECT_ID = None CACHE_CONTROL = 'max-age=172800' + POLICY = 'projectPrivate' # Overriden from settings.FILES_STORE_GCS_ACL in + # FilesPipeline.from_settings. def __init__(self, uri): from google.cloud import storage @@ -239,7 +241,8 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type=self._get_content_type(headers) + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY ) @@ -314,6 +317,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c5fc12afe..5cdddce49 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ead511473..7916b9704 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' +FILES_STORE_GCS_ACL = 'projectPrivate' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' +IMAGES_STORE_GCS_ACL = 'projectPrivate' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 5254ac393bdf712db698bfa41caf5fb2e682f883 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 3 Apr 2018 18:00:08 +0900 Subject: [PATCH 0029/1853] added test for gcs policy --- scrapy/utils/test.py | 3 ++- tests/test_pipeline_files.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 60b931f48..4b935c51b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path): bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted bucket.delete_blob(path) - return content, blob + return content, acl, blob def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c761bd606..728a74803 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase): meta = {'foo': 'bar'} path = 'full/filename' store = GCSFilesStore(uri) + store.POLICY = 'authenticatedRead' + expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) self.assertIn('last_modified', s) self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertIn(expected_policy, acl) class ItemWithFiles(Item): From 74a9c65290888b3ce712c3cfa7132f91ffa3c576 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 3 Apr 2018 18:20:37 +0900 Subject: [PATCH 0030/1853] update docs for support gcs acl --- docs/topics/media-pipeline.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 41beebe98..284ec1a25 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -189,6 +189,8 @@ Google Cloud Storage --------------------- .. setting:: GCS_PROJECT_ID +.. setting:: FILES_STORE_GCS_ACL +.. setting:: IMAGES_STORE_GCS_ACL :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) @@ -204,6 +206,18 @@ For information about authentication, see this `documentation`_. .. _documentation: https://cloud.google.com/docs/authentication/production +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_GCS_ACL` and +:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to +``projectPrivate``. To make the files publicly available use the ``publicRead`` +policy:: + + IMAGES_STORE_GCS_ACL = 'publicRead' + +For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. + +.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + Usage example ============= From cb76b88331e1e0cff30de9a6961de3e28e94ff44 Mon Sep 17 00:00:00 2001 From: grammy-jiang Date: Wed, 4 Apr 2018 05:56:05 -0400 Subject: [PATCH 0031/1853] fix a mistake in topic spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..1d451af21 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -116,7 +116,7 @@ following methods: method (from other spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Response`, dict or + iterable of :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. If it returns ``None``, Scrapy will continue processing this exception, From 464973489e9f457570d7dd1a2e4aab8c4c1778fd Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 12:06:39 +0900 Subject: [PATCH 0032/1853] Using bucket's default object ACL --- docs/topics/media-pipeline.rst | 3 ++- scrapy/pipelines/files.py | 6 ++++-- scrapy/settings/default_settings.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 284ec1a25..0872ac0cd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -209,7 +209,8 @@ For information about authentication, see this `documentation`_. You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to -``projectPrivate``. To make the files publicly available use the ``publicRead`` +None which means that Cloud Storage applies the bucket's default object ACL to the object. +To make the files publicly available use the ``publicRead`` policy:: IMAGES_STORE_GCS_ACL = 'publicRead' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index af1d5488a..8ea70e5d1 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -207,8 +207,10 @@ class GCSFilesStore(object): GCS_PROJECT_ID = None CACHE_CONTROL = 'max-age=172800' - POLICY = 'projectPrivate' # Overriden from settings.FILES_STORE_GCS_ACL in - # FilesPipeline.from_settings. + + # The bucket's default object ACL will be applied to the object. + # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + POLICY = None def __init__(self, uri): from google.cloud import storage diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7916b9704..d7ca8a835 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = 'projectPrivate' +FILES_STORE_GCS_ACL = None FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = 'projectPrivate' +IMAGES_STORE_GCS_ACL = None ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 560ee623fd84c3db986fccacaef7e9a31a8ea02c Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 19:00:27 +0900 Subject: [PATCH 0033/1853] set defalut value "" to FILES_STORE_GCS_ACL --- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 2 +- scrapy/settings/default_settings.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8ea70e5d1..510cc23c7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -319,7 +319,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5cdddce49..95323c613 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,7 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7ca8a835..36e17ef6b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = None +FILES_STORE_GCS_ACL = '' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = None +IMAGES_STORE_GCS_ACL = '' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 6ef6585b5a187d4a8dcec99ba7cae9b6cee91b30 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 19:06:29 +0900 Subject: [PATCH 0034/1853] update docs --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 0872ac0cd..a1f518cbd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -209,7 +209,7 @@ For information about authentication, see this `documentation`_. You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to -None which means that Cloud Storage applies the bucket's default object ACL to the object. +``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object. To make the files publicly available use the ``publicRead`` policy:: From 57b0e6b6955705efcbe2d3da5501b27716ef1baf Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Thu, 19 Apr 2018 13:35:46 +0800 Subject: [PATCH 0035/1853] improve document about functions as processors --- docs/topics/loaders.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index ad86dba63..cdb066e57 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. +If you want to use a plain function as a processor, make sure it receives +``self`` as the first argument:: + + def lowercase_processor(self, values): + for v in values: + yield v.lower() + + class MyItemLoader(ItemLoader): + name_in = lowercase_processor + +This is because whenever a function is assigned as a class variable, it becomes +a method and would be passed the instance as the the first argument when being +called. See `this answer on stackoverflow`_ for more details. + The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -143,6 +157,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. +.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 Declaring Item Loaders ====================== From e75f721c04446f8f28d3bdfcd69f967f65981407 Mon Sep 17 00:00:00 2001 From: Pengyu Chen Date: Mon, 23 Apr 2018 22:08:28 +0800 Subject: [PATCH 0036/1853] Added: Allowing optional arguments for `scrapy.http.cookies.CookieJar.clear` --- scrapy/http/cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a1e95102e..4e8056750 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -58,8 +58,8 @@ class CookieJar(object): def clear_session_cookies(self, *args, **kwargs): return self.jar.clear_session_cookies(*args, **kwargs) - def clear(self): - return self.jar.clear() + def clear(self, domain=None, path=None, name=None): + return self.jar.clear(domain, path, name) def __iter__(self): return iter(self.jar) From 0d015e5c0f0dcb8936044aec163dd12e33482730 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 16 May 2018 09:36:07 +0000 Subject: [PATCH 0037/1853] blacklist twisted version with regression --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..7d857a8c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +Twisted>=13.1.0,!=18.4.0 lxml pyOpenSSL cssselect>=0.9 From c5ddfddb7e5b42e45b6c8f86a381ee52fbb6f797 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 17 May 2018 08:53:42 +0000 Subject: [PATCH 0038/1853] blacklist twisted version with regression in constraints file --- requirements.txt | 2 +- tests/constraints.txt | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/constraints.txt diff --git a/requirements.txt b/requirements.txt index 7d857a8c9..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0,!=18.4.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..3bc30de15 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1 @@ +Twisted!=18.4.0 diff --git a/tox.ini b/tox.ini index 60ff8c15e..c2fa9af28 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27 [testenv] deps = + -ctests/constraints.txt -rrequirements.txt # Extras botocore From 2dfc5d128bba42e2fe2bb24c0326fe47b0d3cd97 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 9 May 2018 11:59:38 -0400 Subject: [PATCH 0039/1853] Update DEPTH_STATS refs to DEPTH_STATS_VERBOSE --- docs/topics/settings.rst | 11 ----------- docs/topics/spider-middleware.rst | 3 ++- scrapy/settings/default_settings.py | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 076dc6bfd..1f1217770 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -335,17 +335,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` and :setting:`RETRY_PRIORITY_ADJUST`. -.. setting:: DEPTH_STATS - -DEPTH_STATS ------------ - -Default: ``True`` - -Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` - -Whether to collect maximum depth stats. - .. setting:: DEPTH_STATS_VERBOSE DEPTH_STATS_VERBOSE diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..265acdb43 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -212,7 +212,8 @@ DepthMiddleware * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to crawl for any site. If zero, no limit will be imposed. - * :setting:`DEPTH_STATS` - Whether to collect depth stats. + * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of + requests for each depth. * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on their depth. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 36e17ef6b..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -55,7 +55,7 @@ DEFAULT_REQUEST_HEADERS = { } DEPTH_LIMIT = 0 -DEPTH_STATS = True +DEPTH_STATS_VERBOSE = False DEPTH_PRIORITY = 0 DNSCACHE_ENABLED = True From 6a182c955273745daf334033944c82da3aa4eb12 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 9 May 2018 12:00:18 -0400 Subject: [PATCH 0040/1853] Depth stats are not optional --- scrapy/spidermiddlewares/depth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index e2f039146..34a87f2df 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class DepthMiddleware(object): - def __init__(self, maxdepth, stats=None, verbose_stats=False, prio=1): + def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth self.stats = stats self.verbose_stats = verbose_stats @@ -41,7 +41,7 @@ class DepthMiddleware(object): extra={'spider': spider} ) return False - elif self.stats: + else: if self.verbose_stats: self.stats.inc_value('request_depth_count/%s' % depth, spider=spider) @@ -50,7 +50,7 @@ class DepthMiddleware(object): return True # base case (depth=0) - if self.stats and 'depth' not in response.meta: + if 'depth' not in response.meta: response.meta['depth'] = 0 if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) From b364d27247b2d9b86c164569c7e0459fa3f8391b Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 23 May 2018 21:25:50 +0300 Subject: [PATCH 0041/1853] [MRG+1] Automatic port selection for servicies in unit tests (#3210) * ability to pass port as a parameter * try to find free ports * use environment variables to pass mock server address * get mock server address from environment variables * ability to select ports for proxy in runtime * use common method for URLs from mock server * https support * get mock server address * get mock address * replace hand-written mechanism by kernel-based one * use ephemeral ports in mockserver * strip EOL from addresses * use ephemeral port in proxy * no need to restore environment as it is restored in tearDown * decode bytes * use mockserver address as a variable * ability to pass address as variable * per test-case mockserver * use base class * remove obsolete environment manipulation * return usage of proxy for http cases * common method for broking proxy auth credentials * python version-independent url methods --- tests/mockserver.py | 25 +++++++++--- tests/spiders.py | 17 ++++++--- tests/test_closespider.py | 8 ++-- tests/test_crawl.py | 48 ++++++++++++------------ tests/test_downloader_handlers.py | 8 ++-- tests/test_feedexport.py | 3 +- tests/test_pipeline_crawl.py | 16 ++++---- tests/test_proxy_connect.py | 41 ++++++++++++-------- tests/test_spidermiddleware_httperror.py | 23 ++++++------ 9 files changed, 111 insertions(+), 78 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 98723846e..f36ce3c44 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -192,9 +192,15 @@ class MockServer(): def __enter__(self): from scrapy.utils.test import get_testenv + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], stdout=PIPE, env=get_testenv()) - self.proc.stdout.readline() + http_address = self.proc.stdout.readline().strip().decode('ascii') + https_address = self.proc.stdout.readline().strip().decode('ascii') + + self.http_address = http_address + self.https_address = https_address + return self def __exit__(self, exc_type, exc_value, traceback): @@ -202,6 +208,12 @@ class MockServer(): self.proc.wait() time.sleep(0.2) + def url(self, path, is_secure=False): + host = self.http_address + if is_secure: + host = self.https_address + return host + path + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'): return ssl.DefaultOpenSSLContextFactory( @@ -213,14 +225,17 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, factory) + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(8999, factory, contextFactory) + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - print("Mock server running at http://%s:%d and https://%s:%d" % ( - httpHost.host, httpHost.port, httpsHost.host, httpsHost.port)) + httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) + httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/spiders.py b/tests/spiders.py index 1038b69de..7816bf7c7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -11,7 +11,12 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor -class MetaSpider(Spider): +class MockServerSpider(Spider): + def __init__(self, mockserver=None, *args, **kwargs): + super(MockServerSpider, self).__init__(*args, **kwargs) + self.mockserver = mockserver + +class MetaSpider(MockServerSpider): name = 'meta' @@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) self.start_urls = [url] def parse(self, response): @@ -55,7 +60,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b) + url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider): return self.errback_func(failure) -class DuplicateStartRequestsSpider(Spider): +class DuplicateStartRequestsSpider(MockServerSpider): dont_filter = True name = 'duplicatestartrequests' distinct_urls = 2 @@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = "http://localhost:8998/echo?headers=1&body=test%d" % i + url = self.mockserver.url("/echo?headers=1&body=test%d" % i) yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index fa0b48998..0eb1b7944 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -18,7 +18,7 @@ class TestCloseSpider(TestCase): def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') itemcount = crawler.stats.get_value('item_scraped_count') @@ -28,7 +28,7 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') pagecount = crawler.stats.get_value('response_received_count') @@ -38,7 +38,7 @@ class TestCloseSpider(TestCase): def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ @@ -50,7 +50,7 @@ class TestCloseSpider(TestCase): def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') stats = crawler.stats diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3fc13eeb7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -26,7 +26,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): crawler = self.runner.create_crawler(FollowAllSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks @@ -42,7 +42,7 @@ class CrawlTestCase(TestCase): def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) - yield crawler.crawl(maxlatency=delay * 2) + yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver) t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) @@ -53,7 +53,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): crawler = self.runner.create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @@ -61,13 +61,13 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - yield crawler.crawl(n=0.5, b=1) + yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) @@ -77,14 +77,14 @@ class CrawlTestCase(TestCase): def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/status?n=503") + yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:65432/status?n=503") + yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -92,14 +92,14 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid./") + yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_before_yield=1) + yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_yielding=1) + yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -121,7 +121,7 @@ class CrawlTestCase(TestCase): def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) #self.assertTrue(False, crawler.spider.seedsseen) #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), # crawler.spider.seedsseen) @@ -130,10 +130,10 @@ class CrawlTestCase(TestCase): def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks @@ -160,7 +160,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) + yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) self.assertEqual(str(l).count("Got response 200"), 1) @defer.inlineCallbacks @@ -168,7 +168,7 @@ with multiples lines # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=0") + yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -176,7 +176,7 @@ with multiples lines # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=1") + yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(l) def _assert_retried(self, log): @@ -186,7 +186,7 @@ with multiples lines @defer.inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1) + req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1) req1 = req0.replace() req2 = req0.replace(headers={'Referer': None}) req3 = req0.replace(headers={'Referer': 'http://example.com'}) @@ -194,7 +194,7 @@ with multiples lines req1.meta['next'] = req2 req2.meta['next'] = req3 crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0) + yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) @@ -220,7 +220,7 @@ with multiples lines est.append(get_engine_status(crawler.engine)) crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) self.assertEqual(s['engine.spider.name'], crawler.spider.name) @@ -244,7 +244,7 @@ with multiples lines raise TestError crawler = self.runner.create_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(), TestError) + yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks @@ -256,7 +256,7 @@ with multiples lines } crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) yield self.assertFailure( - self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @@ -264,13 +264,13 @@ with multiples lines def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200") + yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200") - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503") + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: yield self.runner.join() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b34faa7e7..c91be2c0c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -534,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000})) + yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) failure = crawler.spider.meta['failure'] self.assertIsInstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=Request(url='http://localhost:8998')) + yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] @@ -551,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase): def test_download_gzip_response(self): crawler = get_crawler(SingleRequestSpider) body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', + request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] @@ -560,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase): if six.PY2: request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') + request = request.replace(url=self.mockserver.url('/xpayload')) yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get('failure') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f55927121..0d9f1e83c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase): try: with MockServer() as s: runner = CrawlerRunner(Settings(defaults)) + spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) with open(res_name, 'rb') as f: @@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): for item in items: @@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): pass diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9b81f827d..5985a6f3e 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -46,7 +46,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): def _process_url(self, url): return add_or_replace_parameter( - 'http://localhost:8998/redirect-to', + self.mockserver.url('/redirect-to'), 'goto', url) @@ -134,7 +134,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -143,7 +143,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -152,9 +152,10 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks @@ -165,9 +166,10 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_downloaded(self.items, str(log)) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 6213a51e8..ae1236bcb 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import time +from six.moves.urllib.parse import urlsplit, urlunsplit from threading import Thread from libmproxy import controller, proxy from netlib import http_auth @@ -17,7 +18,7 @@ from tests.mockserver import MockServer class HTTPSProxy(controller.Master, Thread): - def __init__(self, port): + def __init__(self): password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread): server = proxy.ProxyServer(proxy.ProxyConfig( authenticator = authenticator, cacert = cert_path), - port) + 0) + self.server = server Thread.__init__(self) controller.Master.__init__(self, server) + def http_address(self): + return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + + +def _wrong_credentials(proxy_url): + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + return urlunsplit(bad_auth_proxy) class ProxyConnectTestCase(TestCase): @@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy(8888) + + self._proxy = HTTPSProxy() self._proxy.start() + # Wait for the proxy to start. time.sleep(1.0) - os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = self._proxy.http_address() + os.environ['http_proxy'] = self._proxy.http_address() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase): def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) @defer.inlineCallbacks def test_https_noconnect(self): - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): @@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error(l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request("https://localhost:8999/echo") + request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) with LogCapture() as l: yield crawler.crawl(seed=request) @@ -94,10 +105,10 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 19e6bbdcd..dacd0147f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -11,20 +11,21 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.spiders import MockServerSpider -class _HttpErrorSpider(Spider): +class _HttpErrorSpider(MockServerSpider): name = 'httperror' - start_urls = [ - "http://localhost:8998/status?n=200", - "http://localhost:8998/status?n=404", - "http://localhost:8998/status?n=402", - "http://localhost:8998/status?n=500", - ] bypass_status_codes = set() def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) + self.start_urls = [ + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), + ] self.failed = set() self.skipped = set() self.parsed = set() @@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: - yield crawler.crawl(bypass_status_codes={402}) + yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) self.assertEqual(crawler.spider.parsed, {'200', '402'}) self.assertEqual(crawler.spider.skipped, {'402'}) self.assertEqual(crawler.spider.failed, {'404', '500'}) @@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) From ffa7bede17088e5eebd50305704906bf1451ab3a Mon Sep 17 00:00:00 2001 From: Kevin Tewouda Date: Wed, 30 May 2018 06:33:18 +0200 Subject: [PATCH 0042/1853] Update spiders.rst I changed URLs to :class:`~scrapy.http.Request` in start_urls explanation of the default spider --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c2c271245..697732b47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -88,7 +88,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data + listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings From ecdd888ff4614a0e994c9dadb31b7c9c85c88e8a Mon Sep 17 00:00:00 2001 From: Chris Slothouber Date: Fri, 1 Jun 2018 09:25:34 -0400 Subject: [PATCH 0043/1853] Minor edits to contributing.rst Corrected minor grammatical issues and increased clarity of instructions. --- docs/contributing.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f4f9e393f..6615840f7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -48,9 +48,9 @@ guidelines when reporting a new bug. `Stack Overflow `__ (use "scrapy" tag). -* check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report, but check the ticket history and comments. If you - have additional useful information, please leave a comment, or consider +* check the `open issues`_ to see if the issue has already been reported. If it + has, don't dismiss the report, but check the ticket history and comments. If + you have additional useful information, please leave a comment, or consider :ref:`sending a pull request ` with a fix. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has @@ -122,7 +122,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first. Sometimes there is an existing pull request for the problem you'd like to solve, which is stalled for some reason. Often the pull request is in a right direction, but changes are requested by Scrapy maintainers, and the -original pull request author haven't had time to address them. +original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is @@ -143,7 +143,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports -removal, etc) in separate commits than functional changes. This will make pull +removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. Coding style @@ -170,7 +170,7 @@ Documentation policies **do** provide a docstring, but make sure sphinx documentation uses autodoc_ extension to pull the docstring. For example, the :meth:`ItemLoader.add_value` method should be either - documented only in the sphinx documentation (not it a docstring), or + documented only in the sphinx documentation (not as a docstring), or it should have a docstring which is pulled to sphinx documentation using autodoc_ extension. From 6a2d2c3b77bde1d74b46d7dbfb9488cb06e5021f Mon Sep 17 00:00:00 2001 From: Fredrik Bergenlid Date: Fri, 1 Jun 2018 21:38:07 +0200 Subject: [PATCH 0044/1853] Improve gunzip performance for big files --- scrapy/utils/gz.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 16c9ce539..ec3949651 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -30,25 +30,25 @@ def gunzip(data): This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output = b'' + output_list = [] chunk = b'.' while chunk: try: chunk = read1(f, 8196) - output += chunk + output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output is '' and f.extrabuf + # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output or getattr(f, 'extrabuf', None): + if output_list or getattr(f, 'extrabuf', None): try: - output += f.extrabuf[-f.extrasize:] + output_list.append(f.extrabuf[-f.extrasize:]) finally: break else: raise - return output + return b''.join(output_list) _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search From 98d9093dc7241b32b927e79ff2f8a475ae5659a6 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:37:16 -0700 Subject: [PATCH 0045/1853] minor grammatical fixes in CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 162602248..d477168eb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -3,7 +3,7 @@ ## Our Pledge In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and +contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and From 9bd5444a42e82a05d2791fd10a205a9dd99303a6 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:48:28 -0700 Subject: [PATCH 0046/1853] added oxford commas to LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 6ead05ece..4d0a0863a 100644 --- a/LICENSE +++ b/LICENSE @@ -5,10 +5,10 @@ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. + this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the + notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of Scrapy nor the names of its contributors may be used From 12d10eec2cb10fb13cf199d73363093478e80f9a Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:53:53 -0700 Subject: [PATCH 0047/1853] changed Twisted >= 17.9.0 to Twisted>=17.9.0 --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..1f342cfbb 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From 596f39600dfa94504b585ef65b1b04571a631447 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Sat, 19 May 2018 16:32:55 -0700 Subject: [PATCH 0048/1853] reversed earlier change that seemed to cause Travis CI build failure --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 1f342cfbb..2aae3ae65 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted>=17.9.0 +Twisted >= 17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From d4511667fb5df63058accfe731e3d4160e795ee8 Mon Sep 17 00:00:00 2001 From: mugayoshi Date: Sat, 9 Jun 2018 18:17:11 +0900 Subject: [PATCH 0049/1853] Update debugging memory leaks section in the docs Add Python3 tools description. --- docs/topics/leaks.rst | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 92590c180..af14d14e8 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, you still have another resource: the `Guppy library`_. +If you're using Python3, see :ref:`topics-leaks-muppy`. .. _Guppy library: https://pypi.python.org/pypi/guppy @@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the .. _Guppy documentation: http://guppy-pe.sourceforge.net/ +.. _topics-leaks-muppy: + +Debugging memory leaks with muppy +================================= +If you're using Python 3, you can use muppy from `Pympler`_. + +.. _Pympler: https://pypi.org/project/Pympler/ + +If you use ``pip``, you can install muppy with the following command:: + + pip install Pympler + +Here's an example to view all Python objects available in +the heap using muppy:: + + >>> from pympler import muppy + >>> all_objects = muppy.get_objects() + >>> len(all_objects) + 28667 + >>> from pympler import summary + >>> suml = summary.summarize(all_objects) + >>> summary.print_(suml) + types | # objects | total size + ==================================== | =========== | ============ + Date: Wed, 13 Jun 2018 18:11:43 -0300 Subject: [PATCH 0050/1853] Include Python version indication to each required library used in S3 storage --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 135d05c93..b64dbfbfd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ or `boto`_ + * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only) The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: From 72d0899bce06190de5a453b24dd66c8910e6d0ee Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 14 Jun 2018 17:58:48 +0300 Subject: [PATCH 0051/1853] Return non-zero exit code from scrapy commands in case of spider bootstrap errors * method to detect spider creation in crawler * correct method name * method to know if crawlers has spiders * we do not need to issue requests * set exit code accordingly to spiders in crawlers * more portable way to check ofr exceptions * more clear way * test cases for several spiders per crawler * grammatically correct name for method * method is private * grammatically correct name for method * method is private * remove unused import * correct order of imports * changes mechanism of obtaining spider status from method to object member * rename tests --- scrapy/commands/crawl.py | 3 ++ scrapy/commands/runspider.py | 3 ++ scrapy/crawler.py | 2 ++ tests/test_commands.py | 12 +++++++ tests/test_crawler.py | 64 +++++++++++++++++++++++++++++++++++- 5 files changed, 83 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b986bf9d..8093fd402 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -56,3 +56,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a98033dd1..376d3c84e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -87,3 +87,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5cbc2d7c5..04aee18ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -137,6 +137,7 @@ class CrawlerRunner(object): self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() + self.bootstrap_failed = False @property def spiders(self): @@ -178,6 +179,7 @@ class CrawlerRunner(object): def _done(result): self.crawlers.discard(crawler) self._active.discard(d) + self.bootstrap_failed |= not getattr(crawler, 'spider', None) return result return d.addBoth(_done) diff --git a/tests/test_commands.py b/tests/test_commands.py index cb1301c95..7d9071b64 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import inspect import os import sys import subprocess @@ -17,6 +18,7 @@ from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from tests.test_crawler import ExceptionSpider, NoRequestsSpider class ProjectTest(unittest.TestCase): @@ -220,6 +222,16 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_run_fail_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + ret = proc.returncode + self.assertNotEqual(ret, 0) + + def test_run_good_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + ret = proc.returncode + self.assertEqual(ret, 0) + def test_runspider_log_level(self): log = self.get_log(self.debug_log_spider, args=('-s', 'LOG_LEVEL=INFO')) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ba0d709ff..d3b80f460 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,9 @@ import tempfile import warnings import unittest +from twisted.internet import defer +import twisted.trial.unittest + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings @@ -11,9 +14,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle - class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -181,3 +184,62 @@ class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_None(self): runner = CrawlerProcess() self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + + +class ExceptionSpider(scrapy.Spider): + name = 'exception' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + raise ValueError('Exception in from_crawler method') + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful_for_several(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + self.assertEqual(runner.bootstrap_failed, True) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed_for_several(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + yield runner.crawl(NoRequestsSpider) + + self.assertEqual(runner.bootstrap_failed, True) From 7a601d76de7adc37571815a7d08f84e1e26f7507 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 19 Jun 2018 10:51:55 +0200 Subject: [PATCH 0052/1853] fix typo extractred --> extracted --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 1629510b2..1b8d121a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -156,7 +156,7 @@ attributes with ``FormRequest``. **Please also note that link extractors do not canonicalize URLs by default anymore.** This was puzzling users every now and then, and it's not what -browsers do in fact, so we removed that extra transformation on extractred +browsers do in fact, so we removed that extra transformation on extracted links. For those of you wanting more control on the ``Referer:`` header that Scrapy From 88bd067912ee94e2a6d2e1ba5b0d5db1241b7621 Mon Sep 17 00:00:00 2001 From: Grammy Jiang Date: Wed, 20 Jun 2018 16:56:46 +0800 Subject: [PATCH 0053/1853] fix the test case name of HttpProxyMiddleware --- tests/test_downloadermiddleware_httpproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 17be875c1..537126613 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,7 @@ from scrapy.settings import Settings spider = Spider('foo') -class TestDefaultHeadersMiddleware(TestCase): +class TestHttpProxyMiddleware(TestCase): failureException = AssertionError From 9ad3af9d88bcefa18394c1cb2c833902ac20c533 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Sat, 23 Jun 2018 17:31:54 +0800 Subject: [PATCH 0054/1853] Update requirements.txt make the version of ipython less than 6.0 in python 2.7 --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index c1576a2e7..790f29d34 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,4 +10,4 @@ brotlipy testfixtures # optional for shell wrapper tests bpython -ipython +ipython<6.0 From 4740dca8f260bef83eed849b692b3a2c1aaec6cf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 24 Jun 2018 20:59:18 -0300 Subject: [PATCH 0055/1853] Deferred-like process_output/process_exception chain --- scrapy/core/spidermw.py | 58 +++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 50677670b..98e264bd3 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -17,6 +17,11 @@ class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' + # List of dicts. Each dict represents a spider middleware and contains the + # 'process_spider_output', 'process_spider_exception' methods. + # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain + output_methods = [] + @classmethod def _get_mwlist_from_settings(cls, settings): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) @@ -25,12 +30,12 @@ class SpiderMiddlewareManager(MiddlewareManager): super(SpiderMiddlewareManager, self)._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) - if hasattr(mw, 'process_spider_output'): - self.methods['process_spider_output'].insert(0, mw.process_spider_output) - if hasattr(mw, 'process_spider_exception'): - self.methods['process_spider_exception'].insert(0, mw.process_spider_exception) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].insert(0, mw.process_start_requests) + self.output_methods.insert(0, dict( + process_spider_output=getattr(mw, 'process_spider_output', None), + process_spider_exception=getattr(mw, 'process_spider_exception', None), + )) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( @@ -48,45 +53,54 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure): + def process_spider_exception(_failure, mw_index): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for method in self.methods['process_spider_exception']: - result = method(response=response, exception=exception, spider=spider) - if result is not None and not _isiterable(result): - raise _InvalidOutput('Middleware {} must return None or an iterable ' \ - 'object, got {}'.format(fname(method), type(result))) - # stop exception handling by handing control over to the - # process_spider_output chain if an iterable has been returned - if result is not None: - return result + for index, mw in enumerate(self.output_methods): + if index < mw_index or mw['process_spider_exception'] is None: + continue + result = mw['process_spider_exception'](response=response, exception=exception, spider=spider) + mw_index += 1 + if _isiterable(result): + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned + return process_spider_output(result, mw_index) + elif result is None: + continue + else: + raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \ + .format(fname(mw['process_spider_exception']), type(result))) return _failure - def process_spider_output(result): + def process_spider_output(result, mw_index): def wrapper(result_iterable): try: for r in result_iterable: yield r except Exception as ex: - exception_result = process_spider_exception(Failure(ex)) + # process the exception with the method from the next middleware + exception_result = process_spider_exception(Failure(ex), mw_index) if exception_result is None or isinstance(exception_result, Failure): raise for output in exception_result: yield output - for method in self.methods['process_spider_output']: - result = method(response=response, result=result, spider=spider) + for index, mw in enumerate(self.output_methods): + if index < mw_index or mw['process_spider_output'] is None: + continue + result = mw['process_spider_output'](response=response, result=result, spider=spider) + mw_index += 1 if _isiterable(result): result = wrapper(result) else: - raise _InvalidOutput('Middleware {} must return an iterable object, ' \ - 'got {}'.format(fname(method), type(result))) + raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ + .format(fname(mw['process_spider_output']), type(result))) return result dfd = mustbe_deferred(process_spider_input, response) - dfd.addErrback(process_spider_exception) - dfd.addCallback(process_spider_output) + dfd.addErrback(process_spider_exception, mw_index=0) + dfd.addCallback(process_spider_output, mw_index=0) return dfd def process_start_requests(self, start_requests, spider): From ba294351381c0dd81476603246d2cea6c31486be Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 25 Jun 2018 15:01:12 -0300 Subject: [PATCH 0056/1853] Default values for OffsiteMiddleware For some reason test_crawl.py seems to be skipping the spider_opened method, which initializes the host_regex instance variable --- scrapy/spidermiddlewares/offsite.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 310166cad..3b7f194e4 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -19,6 +19,9 @@ class OffsiteMiddleware(object): def __init__(self, stats): self.stats = stats + # default values + self.host_regex = re.compile('') # allow all by default + self.domains_seen = set() @classmethod def from_crawler(cls, crawler): @@ -52,7 +55,7 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return re.compile('') # allow all by default + return url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): @@ -62,8 +65,9 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) - self.domains_seen = set() + host_regex = self.get_host_regex(spider) + if host_regex: + self.host_regex = host_regex class URLWarning(Warning): From fac1b2f3516f3db6ce669664943336c08d95d0aa Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jun 2018 03:23:47 +0500 Subject: [PATCH 0057/1853] TST remove workaround for old Pillow versions which don't support BytesIO --- tests/test_pipeline_images.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 03c6d8059..a7c652959 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,8 +1,8 @@ -import os +import io import hashlib import random import warnings -from tempfile import mkdtemp, TemporaryFile +from tempfile import mkdtemp from shutil import rmtree from twisted.trial import unittest @@ -401,8 +401,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) + def _create_image(format, *a, **kw): - buf = TemporaryFile() + buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf) From 45f67eb64d54f2ac9fcd69233d7bddbdcec88a37 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jun 2018 14:51:01 +0500 Subject: [PATCH 0058/1853] TST exclude lxml==4.2.2 from tests, as it doesn't play well with Pillow --- tests/constraints.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/constraints.txt b/tests/constraints.txt index 3bc30de15..e59e68b3f 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1 +1,2 @@ Twisted!=18.4.0 +lxml!=4.2.2 \ No newline at end of file diff --git a/tox.ini b/tox.ini index c2fa9af28..82348eb24 100644 --- a/tox.ini +++ b/tox.ini @@ -67,6 +67,7 @@ commands = [testenv:py34] basepython = python3.4 deps = + -ctests/constraints.txt -rrequirements-py3.txt # Extras Pillow From 8782901fc865ca6f505bcd1a7d17314076507028 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Jun 2018 01:11:15 +0500 Subject: [PATCH 0059/1853] [MRG+1] TST test agains latest pypy (#3309) pypy3 is not upgraded, as tests segfault with pypy3 6.0 for some reason --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..065f23805 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: install: - | if [ "$TOXENV" = "pypy" ]; then - export PYPY_VERSION="pypy-5.9-linux_x86_64-portable" + export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable" wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" From f11d65f7d66cf2d8560c707c4bb2b76079d45e5f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Jun 2018 18:34:11 +0500 Subject: [PATCH 0060/1853] TST make it clear which requirements are Python 2-only * rename requirements.txt to requirements-py2.txt, to make it clear they are Python 2-only * make requirements-py3.txt consistent with requirements-py2.txt --- requirements.txt => requirements-py2.txt | 4 ++-- requirements-py3.txt | 3 +++ tests/{requirements.txt => requirements-py2.txt} | 0 tox.ini | 8 ++++---- 4 files changed, 9 insertions(+), 6 deletions(-) rename requirements.txt => requirements-py2.txt (100%) rename tests/{requirements.txt => requirements-py2.txt} (100%) diff --git a/requirements.txt b/requirements-py2.txt similarity index 100% rename from requirements.txt rename to requirements-py2.txt index 2a94d742d..03b33d02d 100644 --- a/requirements.txt +++ b/requirements-py2.txt @@ -2,9 +2,9 @@ Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 -w3lib>=1.17.0 queuelib +w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -service_identity parsel>=1.4 +service_identity diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..d76d9412f 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -4,4 +4,7 @@ pyOpenSSL>=0.13.1 cssselect>=0.9 queuelib>=1.1.1 w3lib>=1.17.0 +six>=1.5.2 +PyDispatcher>=2.0.5 +parsel>=1.4 service_identity diff --git a/tests/requirements.txt b/tests/requirements-py2.txt similarity index 100% rename from tests/requirements.txt rename to tests/requirements-py2.txt diff --git a/tox.ini b/tox.ini index 82348eb24..ee40983de 100644 --- a/tox.ini +++ b/tox.ini @@ -9,13 +9,13 @@ envlist = py27 [testenv] deps = -ctests/constraints.txt - -rrequirements.txt + -rrequirements-py2.txt # Extras botocore google-cloud-storage Pillow != 3.0.0 leveldb - -rtests/requirements.txt + -rtests/requirements-py2.txt passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -35,7 +35,7 @@ deps = Pillow==2.3.0 cssselect==0.9.1 zope.interface==4.0.5 - -rtests/requirements.txt + -rtests/requirements-py2.txt [testenv:jessie] # https://packages.debian.org/en/jessie/python/ @@ -50,7 +50,7 @@ deps = Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 - -rtests/requirements.txt + -rtests/requirements-py2.txt [testenv:trunk] basepython = python2.7 From df75a0942e004f9645182a0260769f4337f843e5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 1 Jul 2018 13:30:50 -0300 Subject: [PATCH 0061/1853] Update docs --- docs/topics/spider-middleware.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0af26be73..dde1786af 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -112,9 +112,8 @@ following methods: .. method:: process_spider_exception(response, exception, spider) - This method is called when when a spider or :meth:`process_spider_input`/ - :meth:`process_spider_output` method (from other spider middleware) - raises an exception. + This method is called when a spider or :meth:`process_spider_output` + method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an iterable of :class:`~scrapy.http.Request`, dict or @@ -126,7 +125,7 @@ following methods: exception reaches the engine (where it's logged and discarded). If it returns an iterable the :meth:`process_spider_output` pipeline - kicks in, starting with the last non-executed method, and no other + kicks in, starting from the next spider middleware, and no other :meth:`process_spider_exception` will be called. :param response: the response being processed when the exception was From 735de8167d3e6b0085710d406c8c2976913baa43 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 30 Jun 2018 20:55:17 -0300 Subject: [PATCH 0062/1853] Test for exceptions on process_spider_input --- tests/test_spider_mw.py | 77 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_spider_mw.py diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py new file mode 100644 index 000000000..4a431d379 --- /dev/null +++ b/tests/test_spider_mw.py @@ -0,0 +1,77 @@ + +import logging + +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase +from twisted.internet import defer + +from scrapy import Spider, Request +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + + +class CommonTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def crawl_log(self, spider): + crawler = get_crawler(spider) + with LogCapture() as log: + yield crawler.crawl() + raise defer.returnValue(log) + + +class LogExceptionMiddleware(object): + def process_spider_exception(self, response, exception, spider): + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + return None + + +# ================================================================================ +# (1) exceptions from a spider middleware's process_spider_input method +class ProcessSpiderInputSpider(Spider): + name = 'ProcessSpiderInputSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + __name__ + '.FailProcessSpiderInputMiddleware': 8, + __name__ + '.LogExceptionMiddleware': 6, + # engine + } + } + + def start_requests(self): + yield Request('http://localhost:8998', callback=self.parse, errback=self.errback) + + def parse(self, response): + return [{'test': 1}, {'test': 2}] + + def errback(self, failure): + self.logger.warn('Got a Failure on the Request errback') + + +class FailProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + logging.warn('Middleware: will raise ZeroDivisionError') + raise ZeroDivisionError() + + +class TestProcessSpiderInputSpider(CommonTestCase): + @defer.inlineCallbacks + def test_process_spider_input_errback(self): + """ + (1) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain, it should go directly to the Request errback + """ + log = yield self.crawl_log(ProcessSpiderInputSpider) + self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log)) + self.assertIn('Middleware: will raise ZeroDivisionError', str(log)) + self.assertIn('Got a Failure on the Request errback', str(log)) From 6ed9440ed528ab5c5eece50512e19929e4320b42 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 30 Jun 2018 21:27:10 -0300 Subject: [PATCH 0063/1853] Tests for exceptions on spider callbacks --- tests/test_spider_mw.py | 72 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py index 4a431d379..092546291 100644 --- a/tests/test_spider_mw.py +++ b/tests/test_spider_mw.py @@ -60,11 +60,11 @@ class ProcessSpiderInputSpider(Spider): class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): - logging.warn('Middleware: will raise ZeroDivisionError') - raise ZeroDivisionError() + logging.warn('Middleware: will raise IndexError') + raise IndexError() -class TestProcessSpiderInputSpider(CommonTestCase): +class TestProcessSpiderInput(CommonTestCase): @defer.inlineCallbacks def test_process_spider_input_errback(self): """ @@ -72,6 +72,66 @@ class TestProcessSpiderInputSpider(CommonTestCase): process_spider_exception chain, it should go directly to the Request errback """ log = yield self.crawl_log(ProcessSpiderInputSpider) - self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log)) - self.assertIn('Middleware: will raise ZeroDivisionError', str(log)) - self.assertIn('Got a Failure on the Request errback', str(log)) + self.assertNotIn("Middleware: IndexError exception caught", str(log)) + self.assertIn("Middleware: will raise IndexError", str(log)) + self.assertIn("Got a Failure on the Request errback", str(log)) + + +# ================================================================================ +# (2) exceptions from a spider callback (generator) +class GeneratorCallbackSpider(Spider): + name = 'GeneratorCallbackSpider' + start_urls = ['http://localhost:8998'] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + # engine + }, + } + + def parse(self, response): + yield {'test': 1} + yield {'test': 2} + raise ImportError() + + +class TestGeneratorCallback(CommonTestCase): + @defer.inlineCallbacks + def test_generator_callback(self): + """ + (2) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log = yield self.crawl_log(GeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log)) + self.assertIn("'item_scraped_count': 2", str(log)) + + +# ================================================================================ +# (3) exceptions from a spider callback (not a generator) +class NotAGeneratorCallbackSpider(Spider): + name = 'NotAGeneratorCallbackSpider' + start_urls = ['http://localhost:8998'] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + # engine + }, + } + + def parse(self, response): + return [{'test': 1}, {'test': 1/0}] + + +class TestNotAGeneratorCallback(CommonTestCase): + @defer.inlineCallbacks + def test_not_a_generator_callback(self): + """ + (3) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log = yield self.crawl_log(NotAGeneratorCallbackSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log)) + self.assertNotIn("item_scraped_count", str(log)) From 4fca9aba851133fcdc12bb46c7ae229d9537079a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 1 Jul 2018 13:18:29 -0300 Subject: [PATCH 0064/1853] Recover from a callback exception --- tests/test_spider_mw.py | 137 +++++++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 52 deletions(-) diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py index 092546291..49a60d08b 100644 --- a/tests/test_spider_mw.py +++ b/tests/test_spider_mw.py @@ -10,22 +10,8 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class CommonTestCase(TestCase): - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def crawl_log(self, spider): - crawler = get_crawler(spider) - with LogCapture() as log: - yield crawler.crawl() - raise defer.returnValue(log) +# TEST_URL = 'http://example.org' +TEST_URL = 'http://localhost:8998' class LogExceptionMiddleware(object): @@ -34,6 +20,32 @@ class LogExceptionMiddleware(object): return None +# ================================================================================ +# recover from an exception on a spider callback +class RecoverySpider(Spider): + name = 'RecoverySpider' + start_urls = [TEST_URL] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.RecoveryMiddleware': 10, + }, + } + + def parse(self, response): + yield {'test': 1} + self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) + if not response.meta.get('dont_fail'): + raise ModuleNotFoundError() + +class RecoveryMiddleware(object): + def process_spider_exception(self, response, exception, spider): + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class ProcessSpiderInputSpider(Spider): @@ -49,7 +61,7 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request('http://localhost:8998', callback=self.parse, errback=self.errback) + yield Request(TEST_URL, callback=self.parse, errback=self.errback) def parse(self, response): return [{'test': 1}, {'test': 2}] @@ -64,29 +76,14 @@ class FailProcessSpiderInputMiddleware: raise IndexError() -class TestProcessSpiderInput(CommonTestCase): - @defer.inlineCallbacks - def test_process_spider_input_errback(self): - """ - (1) An exception from the process_spider_input chain should not be caught by the - process_spider_exception chain, it should go directly to the Request errback - """ - log = yield self.crawl_log(ProcessSpiderInputSpider) - self.assertNotIn("Middleware: IndexError exception caught", str(log)) - self.assertIn("Middleware: will raise IndexError", str(log)) - self.assertIn("Got a Failure on the Request errback", str(log)) - - # ================================================================================ # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' - start_urls = ['http://localhost:8998'] + start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { - # spider __name__ + '.LogExceptionMiddleware': 10, - # engine }, } @@ -96,28 +93,14 @@ class GeneratorCallbackSpider(Spider): raise ImportError() -class TestGeneratorCallback(CommonTestCase): - @defer.inlineCallbacks - def test_generator_callback(self): - """ - (2) An exception from a spider's callback should - be caught by the process_spider_exception chain - """ - log = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log)) - self.assertIn("'item_scraped_count': 2", str(log)) - - # ================================================================================ # (3) exceptions from a spider callback (not a generator) class NotAGeneratorCallbackSpider(Spider): name = 'NotAGeneratorCallbackSpider' - start_urls = ['http://localhost:8998'] + start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { - # spider __name__ + '.LogExceptionMiddleware': 10, - # engine }, } @@ -125,13 +108,63 @@ class NotAGeneratorCallbackSpider(Spider): return [{'test': 1}, {'test': 1/0}] -class TestNotAGeneratorCallback(CommonTestCase): +# ================================================================================ +class TestSpiderMiddleware(TestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def crawl_log(self, spider): + crawler = get_crawler(spider) + with LogCapture() as log: + yield crawler.crawl() + raise defer.returnValue(log) + + # @defer.inlineCallbacks + # def test_recovery(self): + # """ + # Recover from an exception from a spider's callback. The final item count should be 3 + # (one from the spider before raising the exception, one from the middleware and one + # from the spider when processing the response that was enqueued from the middleware) + # """ + # log = yield self.crawl_log(RecoverySpider) + # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + # self.assertIn("'item_scraped_count': 3", str(log)) + + @defer.inlineCallbacks + def test_process_spider_input_errback(self): + """ + (1) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain, it should go directly to the Request errback + """ + log1 = yield self.crawl_log(ProcessSpiderInputSpider) + self.assertNotIn("Middleware: IndexError exception caught", str(log1)) + self.assertIn("Middleware: will raise IndexError", str(log1)) + self.assertIn("Got a Failure on the Request errback", str(log1)) + + @defer.inlineCallbacks + def test_generator_callback(self): + """ + (2) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log2 = yield self.crawl_log(GeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks def test_not_a_generator_callback(self): """ (3) An exception from a spider's callback should be caught by the process_spider_exception chain """ - log = yield self.crawl_log(NotAGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log)) - self.assertNotIn("item_scraped_count", str(log)) + log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) + self.assertNotIn("item_scraped_count", str(log3)) From 985ab636cfa0825f100b02583bcfd106d1f4cef6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 1 Jul 2018 17:49:30 -0300 Subject: [PATCH 0065/1853] Store output methods on the 'methods' dict --- scrapy/core/spidermw.py | 43 +++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 98e264bd3..c9dd8c91e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -17,11 +17,6 @@ class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' - # List of dicts. Each dict represents a spider middleware and contains the - # 'process_spider_output', 'process_spider_exception' methods. - # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain - output_methods = [] - @classmethod def _get_mwlist_from_settings(cls, settings): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) @@ -32,10 +27,8 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].insert(0, mw.process_start_requests) - self.output_methods.insert(0, dict( - process_spider_output=getattr(mw, 'process_spider_output', None), - process_spider_exception=getattr(mw, 'process_spider_exception', None), - )) + self.methods['process_spider_output'].insert(0, getattr(mw, 'process_spider_output', None)) + self.methods['process_spider_exception'].insert(0, getattr(mw, 'process_spider_exception', None)) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( @@ -53,54 +46,54 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure, mw_index): + def process_spider_exception(_failure, index): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for index, mw in enumerate(self.output_methods): - if index < mw_index or mw['process_spider_exception'] is None: + for i, method in enumerate(self.methods['process_spider_exception']): + if i < index or method is None: continue - result = mw['process_spider_exception'](response=response, exception=exception, spider=spider) - mw_index += 1 + result = method(response=response, exception=exception, spider=spider) + index += 1 if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return process_spider_output(result, mw_index) + return process_spider_output(result, index) elif result is None: continue else: raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \ - .format(fname(mw['process_spider_exception']), type(result))) + .format(fname(method), type(result))) return _failure - def process_spider_output(result, mw_index): + def process_spider_output(result, index): def wrapper(result_iterable): try: for r in result_iterable: yield r except Exception as ex: # process the exception with the method from the next middleware - exception_result = process_spider_exception(Failure(ex), mw_index) + exception_result = process_spider_exception(Failure(ex), index) if exception_result is None or isinstance(exception_result, Failure): raise for output in exception_result: yield output - for index, mw in enumerate(self.output_methods): - if index < mw_index or mw['process_spider_output'] is None: + for i, method in enumerate(self.methods['process_spider_output']): + if i < index or method is None: continue - result = mw['process_spider_output'](response=response, result=result, spider=spider) - mw_index += 1 + result = method(response=response, result=result, spider=spider) + index += 1 if _isiterable(result): result = wrapper(result) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ - .format(fname(mw['process_spider_output']), type(result))) + .format(fname(method), type(result))) return result dfd = mustbe_deferred(process_spider_input, response) - dfd.addErrback(process_spider_exception, mw_index=0) - dfd.addCallback(process_spider_output, mw_index=0) + dfd.addErrback(process_spider_exception, index=0) + dfd.addCallback(process_spider_output, index=0) return dfd def process_start_requests(self, start_requests, spider): From d05c8677c5079b88db4405fe2bed83dc437c9204 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Wed, 4 Jul 2018 02:58:43 +0800 Subject: [PATCH 0066/1853] [MRG+1] change the bad smell code (#3304) Change the bad smell code --- scrapy/downloadermiddlewares/httpproxy.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 0d5320bf8..1dd47359f 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,14 +1,13 @@ import base64 +from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -from six.moves.urllib.parse import unquote try: from urllib2 import _parse_proxy except ImportError: from urllib.request import _parse_proxy -from six.moves.urllib.parse import urlunparse -from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -17,8 +16,8 @@ class HttpProxyMiddleware(object): def __init__(self, auth_encoding='latin-1'): self.auth_encoding = auth_encoding self.proxies = {} - for type, url in getproxies().items(): - self.proxies[type] = self._get_proxy(url, type) + for type_, url in getproxies().items(): + self.proxies[type_] = self._get_proxy(url, type_) @classmethod def from_crawler(cls, crawler): From 74ce1561542dac9be5d1363a4a3e623653855e05 Mon Sep 17 00:00:00 2001 From: chainly <1258626769@qq.com> Date: Wed, 4 Jul 2018 03:00:59 +0800 Subject: [PATCH 0067/1853] add item_error to be catchable (#3256) --- docs/topics/signals.rst | 23 +++++++++++++++++++++++ scrapy/core/scraper.py | 3 +++ scrapy/signals.py | 1 + tests/pipelines.py | 6 ++++++ tests/test_engine.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index cf1588df8..d40c0e1df 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -135,6 +135,29 @@ item_dropped to be dropped :type exception: :exc:`~scrapy.exceptions.DropItem` exception +item_error +------------ + +.. signal:: item_error +.. function:: item_error(item, response, spider, failure) + + Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises + an exception), except :exc:`~scrapy.exceptions.DropItem` exception. + + This signal supports returning deferreds from their handlers. + + :param item: the item dropped from the :ref:`topics-item-pipeline` + :type item: dict or :class:`~scrapy.item.Item` object + + :param response: the response being processed when the exception was raised + :type response: :class:`~scrapy.http.Response` object + + :param spider: the spider which raised the exception + :type spider: :class:`~scrapy.spiders.Spider` object + + :param failure: the exception raised as a Twisted `Failure`_ object + :type failure: `Failure`_ object + spider_closed ------------- diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c08e37367..ee1e95a0c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -232,6 +232,9 @@ class Scraper(object): logger.error('Error processing %(item)s', {'item': item}, exc_info=failure_to_exc_info(output), extra={'spider': spider}) + return self.signals.send_catch_log_deferred( + signal=signals.item_error, item=item, response=response, + spider=spider, failure=output) else: logkws = self.logformatter.scraped(output, response, spider) logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) diff --git a/scrapy/signals.py b/scrapy/signals.py index de0886fb6..e36c27203 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ response_received = object() response_downloaded = object() item_scraped = object() item_dropped = object() +item_error = object() # for backwards compatibility stats_spider_opened = spider_opened diff --git a/tests/pipelines.py b/tests/pipelines.py index ddfbc7a99..7e2895a5c 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -9,3 +9,9 @@ class ZeroDivisionErrorPipeline(object): def process_item(self, item, spider): return item + + +class ProcessWithZeroDivisionErrorPipiline(object): + + def process_item(self, item, spider): + 1/0 diff --git a/tests/test_engine.py b/tests/test_engine.py index 04113ddcf..719c0c60c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -74,6 +74,14 @@ class DictItemsSpider(TestSpider): item_cls = dict +class ItemZeroDivisionErrorSpider(TestSpider): + custom_settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300, + } + } + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -95,6 +103,7 @@ class CrawlerRun(object): self.respplug = [] self.reqplug = [] self.reqdropped = [] + self.itemerror = [] self.itemresp = [] self.signals_catched = {} self.spider_class = spider_class @@ -112,6 +121,7 @@ class CrawlerRun(object): self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.item_error, signals.item_error) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) @@ -136,6 +146,9 @@ class CrawlerRun(object): u = urlparse(url) return u.path + def item_error(self, item, response, spider, failure): + self.itemerror.append((item, response, spider, failure)) + def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) @@ -175,6 +188,10 @@ class EngineTest(unittest.TestCase): self._assert_scheduled_requests(urls_to_visit=7) self._assert_dropped_requests() + self.run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield self.run.run() + self._assert_items_error() + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] @@ -209,6 +226,20 @@ class EngineTest(unittest.TestCase): if self.run.getpath(response.url) == '/redirect': self.assertEqual(302, response.status) + def _assert_items_error(self): + self.assertEqual(2, len(self.run.itemerror)) + for item, response, spider, failure in self.run.itemerror: + self.assertEqual(failure.value.__class__, ZeroDivisionError) + self.assertEqual(spider, self.run.spider) + + self.assertEqual(item['url'], response.url) + if 'item1.html' in item['url']: + self.assertEqual('Item 1 name', item['name']) + self.assertEqual('100', item['price']) + if 'item2.html' in item['url']: + self.assertEqual('Item 2 name', item['name']) + self.assertEqual('200', item['price']) + def _assert_scraped_items(self): self.assertEqual(2, len(self.run.itemresp)) for item, response in self.run.itemresp: From 6f5c39d65f3e5d74c292f2143b4f84c7f43b155a Mon Sep 17 00:00:00 2001 From: Oz T Date: Wed, 4 Jul 2018 00:22:24 +0300 Subject: [PATCH 0068/1853] Fix for CSV export unnecessary blank lines problem on Windows (#3039) --- scrapy/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 07f43b494..695c74fec 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -214,7 +214,8 @@ class CsvItemExporter(BaseItemExporter): file, line_buffering=False, write_through=True, - encoding=self.encoding + encoding=self.encoding, + newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 ) if six.PY3 else file self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True From 0b2870634af6cc14191faafcba7d58a5f3cc3016 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 4 Jul 2018 16:14:51 -0300 Subject: [PATCH 0069/1853] Do not inherit from object --- tests/test_spider_mw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py index 49a60d08b..2565ef7af 100644 --- a/tests/test_spider_mw.py +++ b/tests/test_spider_mw.py @@ -14,7 +14,7 @@ from tests.mockserver import MockServer TEST_URL = 'http://localhost:8998' -class LogExceptionMiddleware(object): +class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): logging.warn('Middleware: %s exception caught', exception.__class__.__name__) return None @@ -37,7 +37,7 @@ class RecoverySpider(Spider): if not response.meta.get('dont_fail'): raise ModuleNotFoundError() -class RecoveryMiddleware(object): +class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): logging.warn('Middleware: %s exception caught', exception.__class__.__name__) return [ From 0a0e62272e67aaebe29666017e9b0623b81bf369 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 4 Jul 2018 16:19:19 -0300 Subject: [PATCH 0070/1853] New tests --- tests/test_spider_mw.py | 170 -------------- tests/test_spidermiddleware.py | 405 ++++++++++----------------------- 2 files changed, 118 insertions(+), 457 deletions(-) delete mode 100644 tests/test_spider_mw.py diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py deleted file mode 100644 index 2565ef7af..000000000 --- a/tests/test_spider_mw.py +++ /dev/null @@ -1,170 +0,0 @@ - -import logging - -from testfixtures import LogCapture -from twisted.trial.unittest import TestCase -from twisted.internet import defer - -from scrapy import Spider, Request -from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer - - -# TEST_URL = 'http://example.org' -TEST_URL = 'http://localhost:8998' - - -class LogExceptionMiddleware: - def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) - return None - - -# ================================================================================ -# recover from an exception on a spider callback -class RecoverySpider(Spider): - name = 'RecoverySpider' - start_urls = [TEST_URL] - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.RecoveryMiddleware': 10, - }, - } - - def parse(self, response): - yield {'test': 1} - self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) - if not response.meta.get('dont_fail'): - raise ModuleNotFoundError() - -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - -# ================================================================================ -# (1) exceptions from a spider middleware's process_spider_input method -class ProcessSpiderInputSpider(Spider): - name = 'ProcessSpiderInputSpider' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # spider - __name__ + '.LogExceptionMiddleware': 10, - __name__ + '.FailProcessSpiderInputMiddleware': 8, - __name__ + '.LogExceptionMiddleware': 6, - # engine - } - } - - def start_requests(self): - yield Request(TEST_URL, callback=self.parse, errback=self.errback) - - def parse(self, response): - return [{'test': 1}, {'test': 2}] - - def errback(self, failure): - self.logger.warn('Got a Failure on the Request errback') - - -class FailProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): - logging.warn('Middleware: will raise IndexError') - raise IndexError() - - -# ================================================================================ -# (2) exceptions from a spider callback (generator) -class GeneratorCallbackSpider(Spider): - name = 'GeneratorCallbackSpider' - start_urls = [TEST_URL] - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, - }, - } - - def parse(self, response): - yield {'test': 1} - yield {'test': 2} - raise ImportError() - - -# ================================================================================ -# (3) exceptions from a spider callback (not a generator) -class NotAGeneratorCallbackSpider(Spider): - name = 'NotAGeneratorCallbackSpider' - start_urls = [TEST_URL] - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, - }, - } - - def parse(self, response): - return [{'test': 1}, {'test': 1/0}] - - -# ================================================================================ -class TestSpiderMiddleware(TestCase): - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def crawl_log(self, spider): - crawler = get_crawler(spider) - with LogCapture() as log: - yield crawler.crawl() - raise defer.returnValue(log) - - # @defer.inlineCallbacks - # def test_recovery(self): - # """ - # Recover from an exception from a spider's callback. The final item count should be 3 - # (one from the spider before raising the exception, one from the middleware and one - # from the spider when processing the response that was enqueued from the middleware) - # """ - # log = yield self.crawl_log(RecoverySpider) - # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - # self.assertIn("'item_scraped_count': 3", str(log)) - - @defer.inlineCallbacks - def test_process_spider_input_errback(self): - """ - (1) An exception from the process_spider_input chain should not be caught by the - process_spider_exception chain, it should go directly to the Request errback - """ - log1 = yield self.crawl_log(ProcessSpiderInputSpider) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - - @defer.inlineCallbacks - def test_generator_callback(self): - """ - (2) An exception from a spider's callback should - be caught by the process_spider_exception chain - """ - log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) - - @defer.inlineCallbacks - def test_not_a_generator_callback(self): - """ - (3) An exception from a spider's callback should - be caught by the process_spider_exception chain - """ - log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 109bcc250..2565ef7af 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -5,256 +5,111 @@ from testfixtures import LogCapture from twisted.trial.unittest import TestCase from twisted.internet import defer -from scrapy.spiders import Spider -from scrapy.item import Item, Field -from scrapy.http import Request +from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class TestItem(Item): - value = Field() +# TEST_URL = 'http://example.org' +TEST_URL = 'http://localhost:8998' -class LocalhostSpider(Spider): - start_urls = ['http://localhost:8998'] # tests.mockserver.MockServer - - -# ================================================================================ -# exceptions from a spider's parse method -class BaseExceptionFromParseMethodSpider(LocalhostSpider): - custom_settings = { - 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540} - } - - -class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider): - """ return value is NOT a generator """ - name = 'not_a_generator' - - def parse(self, response): - raise AssertionError - - -class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider): - """ return value is a generator; the exception is raised - before the items are yielded: no items should be scraped """ - name = 'generator_error_before_items' - - def parse(self, response): - raise ValueError - for i in range(3): - yield {'value': i} - - -class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider): - """ return value is a generator; the exception is raised - after the items are yielded: 3 items should be scraped """ - name = 'generator_error_after_items' - - def parse(self, response): - for i in range(3): - yield {'value': i} - raise FloatingPointError - - -class CatchExceptionMiddleware(object): +class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): - """ catch an exception and log it """ - logging.warn('{} exception caught'.format(exception.__class__.__name__)) + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) return None # ================================================================================ -# exception from a previous middleware's process_spider_input method -# process_spider_input is not expected to return an iterable, so there are no -# separate tests for generator/non-generator implementations -class FromPreviousMiddlewareInputSpider(LocalhostSpider): - name = 'not_a_generator_from_previous_middleware_input' +# recover from an exception on a spider callback +class RecoverySpider(Spider): + name = 'RecoverySpider' + start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return None - - -class RaiseExceptionOnInputMiddleware(object): - def process_spider_input(self, response, spider): - raise LookupError - - -# ================================================================================ -# exception from a previous middleware's process_spider_output method (not a generator) -class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): - name = 'not_a_generator_from_previous_middleware_output' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return [{'value': i} for i in range(3)] - - -class RaiseExceptionOnOutputNotAGeneratorMiddleware(object): - def process_spider_output(self, response, result, spider): - raise UnicodeError - - -# ================================================================================ -# exception from a previous middleware's process_spider_output method (generator) -class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): - name = 'generator_from_previous_middleware_output' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return [{'value': i} for i in range(10, 13)] - - -class RaiseExceptionOnOutputGeneratorMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - yield r - raise NameError - - -# ================================================================================ -# do something useful from the exception handler -class DoSomethingSpider(LocalhostSpider): - name = 'do_something' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.DoSomethingMiddleware': 540, - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, - # spider side - } - } - - def parse(self, response): - yield {'value': response.url} - raise ImportError - - -class DoSomethingMiddleware(object): - def process_spider_exception(self, response, exception, spider): - return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')] - - -# ================================================================================ -# don't catch _InvalidOutput from scrapy's spider middleware manager -class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): - name = 'invalid_return_value_from_previous_middleware_input' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540, - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return None - - -class InvalidReturnValueInputMiddleware(object): - def process_spider_input(self, response, spider): - return 1.0 # , not None - - -class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider): - name = 'invalid_return_value_from_previous_middleware_output' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return None - - -class InvalidReturnValueOutputMiddleware(object): - def process_spider_output(self, response, result, spider): - return 1 # , not an iterable - - -# ================================================================================ -# make sure only non already called process_spider_output methods -# are called if process_spider_exception returns an iterable -class ExecutionChainSpider(LocalhostSpider): - name = 'execution_chain' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.ThirdMiddleware': 540, - 'tests.test_spidermiddleware.SecondMiddleware': 541, - 'tests.test_spidermiddleware.FirstMiddleware': 542 - # spider side + __name__ + '.RecoveryMiddleware': 10, }, } def parse(self, response): - return None - - -class FirstMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - if isinstance(r, dict): - r['handled_by_first_middleware'] = True - yield r + yield {'test': 1} + self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) + if not response.meta.get('dont_fail'): + raise ModuleNotFoundError() +class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): - # log exception, handle control to the next middleware's process_spider_exception - logging.warn('{} exception caught'.format(exception.__class__.__name__)) - return None + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] -class SecondMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - if isinstance(r, dict): - r['handled_by_second_middleware'] = True - yield r - raise MemoryError +# ================================================================================ +# (1) exceptions from a spider middleware's process_spider_input method +class ProcessSpiderInputSpider(Spider): + name = 'ProcessSpiderInputSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + __name__ + '.FailProcessSpiderInputMiddleware': 8, + __name__ + '.LogExceptionMiddleware': 6, + # engine + } + } + + def start_requests(self): + yield Request(TEST_URL, callback=self.parse, errback=self.errback) + + def parse(self, response): + return [{'test': 1}, {'test': 2}] + + def errback(self, failure): + self.logger.warn('Got a Failure on the Request errback') -class ThirdMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - if isinstance(r, dict): - r['handled_by_third_middleware'] = True - yield r - - def process_spider_exception(self, response, exception, spider): - # handle control to the next middleware's process_spider_output - return [{'item': i} for i in range(3)] +class FailProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + logging.warn('Middleware: will raise IndexError') + raise IndexError() +# ================================================================================ +# (2) exceptions from a spider callback (generator) +class GeneratorCallbackSpider(Spider): + name = 'GeneratorCallbackSpider' + start_urls = [TEST_URL] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.LogExceptionMiddleware': 10, + }, + } + + def parse(self, response): + yield {'test': 1} + yield {'test': 2} + raise ImportError() + + +# ================================================================================ +# (3) exceptions from a spider callback (not a generator) +class NotAGeneratorCallbackSpider(Spider): + name = 'NotAGeneratorCallbackSpider' + start_urls = [TEST_URL] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.LogExceptionMiddleware': 10, + }, + } + + def parse(self, response): + return [{'test': 1}, {'test': 1/0}] + + +# ================================================================================ class TestSpiderMiddleware(TestCase): - @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -263,7 +118,7 @@ class TestSpiderMiddleware(TestCase): @classmethod def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - + @defer.inlineCallbacks def crawl_log(self, spider): crawler = get_crawler(spider) @@ -271,69 +126,45 @@ class TestSpiderMiddleware(TestCase): yield crawler.crawl() raise defer.returnValue(log) - @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method_non_generator(self): - # non-generator return value - log = yield self.crawl_log(NotAGeneratorSpider) - self.assertIn("AssertionError exception caught", str(log)) - self.assertIn("spider_exceptions/AssertionError", str(log)) + # @defer.inlineCallbacks + # def test_recovery(self): + # """ + # Recover from an exception from a spider's callback. The final item count should be 3 + # (one from the spider before raising the exception, one from the middleware and one + # from the spider when processing the response that was enqueued from the middleware) + # """ + # log = yield self.crawl_log(RecoverySpider) + # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + # self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method_generator_no_items(self): - # generator return value, no items before the error - log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider) - self.assertIn("ValueError exception caught", str(log)) - self.assertIn("spider_exceptions/ValueError", str(log)) - + def test_process_spider_input_errback(self): + """ + (1) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain, it should go directly to the Request errback + """ + log1 = yield self.crawl_log(ProcessSpiderInputSpider) + self.assertNotIn("Middleware: IndexError exception caught", str(log1)) + self.assertIn("Middleware: will raise IndexError", str(log1)) + self.assertIn("Got a Failure on the Request errback", str(log1)) + @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method_generator_with_items(self): - # generator return value, 3 items before the error - log = yield self.crawl_log(GeneratorErrorAfterItemsSpider) - self.assertIn("'item_scraped_count': 3", str(log)) - self.assertIn("FloatingPointError exception caught", str(log)) - self.assertIn("spider_exceptions/FloatingPointError", str(log)) - + def test_generator_callback(self): + """ + (2) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log2 = yield self.crawl_log(GeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks - def test_process_spider_exception_from_previous_middleware_input(self): - log = yield self.crawl_log(FromPreviousMiddlewareInputSpider) - self.assertIn("LookupError exception caught", str(log)) - - @defer.inlineCallbacks - def test_process_spider_exception_from_previous_middleware_output(self): - # non-generator output value - log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider) - self.assertNotIn("UnicodeError exception caught", str(log)) - # generator output value - log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider) - self.assertIn("'item_scraped_count': 3", str(log)) - self.assertIn("NameError exception caught", str(log)) - - @defer.inlineCallbacks - def test_process_spider_exception_do_something(self): - log = yield self.crawl_log(DoSomethingSpider) - self.assertIn("ImportError exception caught", str(log)) - self.assertIn("{'value': 10}", str(log)) - self.assertIn("{'value': 'asdf'}", str(log)) - self.assertIn("{'value': 'http://localhost:8998'}", str(log)) - self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log)) - - @defer.inlineCallbacks - def test_process_spider_exception_invalid_return_value_previous_middleware(self): - """ don't catch _InvalidOutput from middleware """ - # on middleware's input - log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider) - self.assertNotIn("_InvalidOutput exception caught", str(log1)) - self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) - # on middleware's output - log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider) - self.assertNotIn("_InvalidOutput exception caught", str(log2)) - self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) - - @defer.inlineCallbacks - def test_process_spider_exception_execution_chain(self): - # on middleware's input - log = yield self.crawl_log(ExecutionChainSpider) - self.assertNotIn("handled_by_first_middleware", str(log)) - self.assertNotIn("handled_by_second_middleware", str(log)) - self.assertIn("MemoryError exception caught", str(log)) - self.assertIn("handled_by_third_middleware", str(log)) + def test_not_a_generator_callback(self): + """ + (3) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) + self.assertNotIn("item_scraped_count", str(log3)) From d4c7cc848b83c2ec38ea90b76369885daba7375c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:19:43 +0500 Subject: [PATCH 0071/1853] remove backwards compatibility shims for relocated modules --- .coveragerc | 9 --------- conftest.py | 9 +-------- scrapy/command.py | 7 ------- scrapy/contrib/__init__.py | 0 scrapy/contrib/closespider.py | 7 ------- scrapy/contrib/corestats.py | 7 ------- scrapy/contrib/debug.py | 7 ------- .../contrib/downloadermiddleware/__init__.py | 0 .../contrib/downloadermiddleware/ajaxcrawl.py | 7 ------- scrapy/contrib/downloadermiddleware/chunked.py | 7 ------- scrapy/contrib/downloadermiddleware/cookies.py | 7 ------- .../downloadermiddleware/decompression.py | 7 ------- .../downloadermiddleware/defaultheaders.py | 7 ------- .../downloadermiddleware/downloadtimeout.py | 7 ------- .../contrib/downloadermiddleware/httpauth.py | 7 ------- .../contrib/downloadermiddleware/httpcache.py | 7 ------- .../downloadermiddleware/httpcompression.py | 7 ------- .../contrib/downloadermiddleware/httpproxy.py | 7 ------- .../contrib/downloadermiddleware/redirect.py | 7 ------- scrapy/contrib/downloadermiddleware/retry.py | 7 ------- .../contrib/downloadermiddleware/robotstxt.py | 7 ------- scrapy/contrib/downloadermiddleware/stats.py | 7 ------- .../contrib/downloadermiddleware/useragent.py | 7 ------- scrapy/contrib/exporter/__init__.py | 8 -------- scrapy/contrib/feedexport.py | 7 ------- scrapy/contrib/httpcache.py | 7 ------- scrapy/contrib/linkextractors/__init__.py | 7 ------- scrapy/contrib/linkextractors/htmlparser.py | 7 ------- scrapy/contrib/linkextractors/lxmlhtml.py | 7 ------- scrapy/contrib/linkextractors/regex.py | 7 ------- scrapy/contrib/linkextractors/sgml.py | 7 ------- scrapy/contrib/loader/__init__.py | 7 ------- scrapy/contrib/loader/common.py | 7 ------- scrapy/contrib/loader/processor.py | 7 ------- scrapy/contrib/logstats.py | 7 ------- scrapy/contrib/memdebug.py | 7 ------- scrapy/contrib/memusage.py | 7 ------- scrapy/contrib/pipeline/__init__.py | 7 ------- scrapy/contrib/pipeline/files.py | 7 ------- scrapy/contrib/pipeline/images.py | 7 ------- scrapy/contrib/pipeline/media.py | 7 ------- scrapy/contrib/spidermiddleware/__init__.py | 0 scrapy/contrib/spidermiddleware/depth.py | 7 ------- scrapy/contrib/spidermiddleware/httperror.py | 7 ------- scrapy/contrib/spidermiddleware/offsite.py | 7 ------- scrapy/contrib/spidermiddleware/referer.py | 7 ------- scrapy/contrib/spidermiddleware/urllength.py | 7 ------- scrapy/contrib/spiders/__init__.py | 7 ------- scrapy/contrib/spiders/crawl.py | 7 ------- scrapy/contrib/spiders/feed.py | 7 ------- scrapy/contrib/spiders/init.py | 7 ------- scrapy/contrib/spiders/sitemap.py | 7 ------- scrapy/contrib/spiderstate.py | 7 ------- scrapy/contrib/statsmailer.py | 7 ------- scrapy/contrib/throttle.py | 7 ------- scrapy/contrib_exp/__init__.py | 0 .../downloadermiddleware/__init__.py | 0 .../downloadermiddleware/decompression.py | 7 ------- scrapy/contrib_exp/iterators.py | 6 ------ scrapy/dupefilter.py | 7 ------- scrapy/linkextractor.py | 7 ------- scrapy/spider.py | 7 ------- scrapy/squeue.py | 7 ------- scrapy/statscol.py | 7 ------- scrapy/utils/decorator.py | 7 ------- scrapy/utils/deprecate.py | 18 ------------------ 66 files changed, 1 insertion(+), 441 deletions(-) delete mode 100644 scrapy/command.py delete mode 100644 scrapy/contrib/__init__.py delete mode 100644 scrapy/contrib/closespider.py delete mode 100644 scrapy/contrib/corestats.py delete mode 100644 scrapy/contrib/debug.py delete mode 100644 scrapy/contrib/downloadermiddleware/__init__.py delete mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawl.py delete mode 100644 scrapy/contrib/downloadermiddleware/chunked.py delete mode 100644 scrapy/contrib/downloadermiddleware/cookies.py delete mode 100644 scrapy/contrib/downloadermiddleware/decompression.py delete mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py delete mode 100644 scrapy/contrib/downloadermiddleware/downloadtimeout.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpauth.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpcompression.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpproxy.py delete mode 100644 scrapy/contrib/downloadermiddleware/redirect.py delete mode 100644 scrapy/contrib/downloadermiddleware/retry.py delete mode 100644 scrapy/contrib/downloadermiddleware/robotstxt.py delete mode 100644 scrapy/contrib/downloadermiddleware/stats.py delete mode 100644 scrapy/contrib/downloadermiddleware/useragent.py delete mode 100644 scrapy/contrib/exporter/__init__.py delete mode 100644 scrapy/contrib/feedexport.py delete mode 100644 scrapy/contrib/httpcache.py delete mode 100644 scrapy/contrib/linkextractors/__init__.py delete mode 100644 scrapy/contrib/linkextractors/htmlparser.py delete mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py delete mode 100644 scrapy/contrib/linkextractors/regex.py delete mode 100644 scrapy/contrib/linkextractors/sgml.py delete mode 100644 scrapy/contrib/loader/__init__.py delete mode 100644 scrapy/contrib/loader/common.py delete mode 100644 scrapy/contrib/loader/processor.py delete mode 100644 scrapy/contrib/logstats.py delete mode 100644 scrapy/contrib/memdebug.py delete mode 100644 scrapy/contrib/memusage.py delete mode 100644 scrapy/contrib/pipeline/__init__.py delete mode 100644 scrapy/contrib/pipeline/files.py delete mode 100644 scrapy/contrib/pipeline/images.py delete mode 100644 scrapy/contrib/pipeline/media.py delete mode 100644 scrapy/contrib/spidermiddleware/__init__.py delete mode 100644 scrapy/contrib/spidermiddleware/depth.py delete mode 100644 scrapy/contrib/spidermiddleware/httperror.py delete mode 100644 scrapy/contrib/spidermiddleware/offsite.py delete mode 100644 scrapy/contrib/spidermiddleware/referer.py delete mode 100644 scrapy/contrib/spidermiddleware/urllength.py delete mode 100644 scrapy/contrib/spiders/__init__.py delete mode 100644 scrapy/contrib/spiders/crawl.py delete mode 100644 scrapy/contrib/spiders/feed.py delete mode 100644 scrapy/contrib/spiders/init.py delete mode 100644 scrapy/contrib/spiders/sitemap.py delete mode 100644 scrapy/contrib/spiderstate.py delete mode 100644 scrapy/contrib/statsmailer.py delete mode 100644 scrapy/contrib/throttle.py delete mode 100644 scrapy/contrib_exp/__init__.py delete mode 100644 scrapy/contrib_exp/downloadermiddleware/__init__.py delete mode 100644 scrapy/contrib_exp/downloadermiddleware/decompression.py delete mode 100644 scrapy/contrib_exp/iterators.py delete mode 100644 scrapy/dupefilter.py delete mode 100644 scrapy/linkextractor.py delete mode 100644 scrapy/spider.py delete mode 100644 scrapy/squeue.py delete mode 100644 scrapy/statscol.py delete mode 100644 scrapy/utils/decorator.py diff --git a/.coveragerc b/.coveragerc index 3105409ba..aeadccb25 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,13 +7,4 @@ omit = scrapy/conf.py scrapy/stats.py scrapy/project.py - scrapy/utils/decorator.py - scrapy/statscol.py - scrapy/squeue.py scrapy/log.py - scrapy/dupefilter.py - scrapy/command.py - scrapy/linkextractor.py - scrapy/spider.py - scrapy/contrib/* - scrapy/contrib_exp/* diff --git a/conftest.py b/conftest.py index 8b4faf8fc..c733db646 100644 --- a/conftest.py +++ b/conftest.py @@ -13,19 +13,12 @@ collect_ignore = [ "scrapy/conf.py", "scrapy/stats.py", "scrapy/project.py", - "scrapy/utils/decorator.py", - "scrapy/statscol.py", - "scrapy/squeue.py", "scrapy/log.py", - "scrapy/dupefilter.py", - "scrapy/command.py", - "scrapy/linkextractor.py", - "scrapy/spider.py", # not a test, but looks like a test "scrapy/utils/testsite.py", -] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +] if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): collect_ignore += _py_files("scrapy/xlib/tx") diff --git a/scrapy/command.py b/scrapy/command.py deleted file mode 100644 index 3e1219bbc..000000000 --- a/scrapy/command.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.command` is deprecated, " - "use `scrapy.commands` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.commands import * diff --git a/scrapy/contrib/__init__.py b/scrapy/contrib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py deleted file mode 100644 index 9c52c418f..000000000 --- a/scrapy/contrib/closespider.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.closespider` is deprecated, " - "use `scrapy.extensions.closespider` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.closespider import * diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py deleted file mode 100644 index 2f5354239..000000000 --- a/scrapy/contrib/corestats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.corestats` is deprecated, " - "use `scrapy.extensions.corestats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.corestats import * diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py deleted file mode 100644 index a38f059ce..000000000 --- a/scrapy/contrib/debug.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.debug` is deprecated, " - "use `scrapy.extensions.debug` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.debug import * diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py deleted file mode 100644 index 90ebc46b6..000000000 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, " - "use `scrapy.downloadermiddlewares.ajaxcrawl` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.ajaxcrawl import * diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py deleted file mode 100644 index 1322c9083..000000000 --- a/scrapy/contrib/downloadermiddleware/chunked.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, " - "use `scrapy.downloadermiddlewares.chunked` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.chunked import * diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py deleted file mode 100644 index bad970690..000000000 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, " - "use `scrapy.downloadermiddlewares.cookies` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.cookies import * diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py deleted file mode 100644 index a541aa61e..000000000 --- a/scrapy/contrib/downloadermiddleware/decompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.downloadermiddlewares.decompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.decompression import * diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py deleted file mode 100644 index cf023dc8f..000000000 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, " - "use `scrapy.downloadermiddlewares.defaultheaders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.defaultheaders import * diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py deleted file mode 100644 index 84bd06acf..000000000 --- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, " - "use `scrapy.downloadermiddlewares.downloadtimeout` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.downloadtimeout import * diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py deleted file mode 100644 index a37ffa0dc..000000000 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.httpauth` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpauth import * diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py deleted file mode 100644 index f5f068204..000000000 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, " - "use `scrapy.downloadermiddlewares.httpcache` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpcache import * diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py deleted file mode 100644 index 8a52ec50b..000000000 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, " - "use `scrapy.downloadermiddlewares.httpcompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpcompression import * diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py deleted file mode 100644 index d94d85076..000000000 --- a/scrapy/contrib/downloadermiddleware/httpproxy.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, " - "use `scrapy.downloadermiddlewares.httpproxy` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpproxy import * diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py deleted file mode 100644 index 824eee8ae..000000000 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, " - "use `scrapy.downloadermiddlewares.redirect` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.redirect import * diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py deleted file mode 100644 index aafe0f508..000000000 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, " - "use `scrapy.downloadermiddlewares.retry` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.retry import * diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py deleted file mode 100644 index 408f760a0..000000000 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, " - "use `scrapy.downloadermiddlewares.robotstxt` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.robotstxt import * diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py deleted file mode 100644 index fa84a8206..000000000 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, " - "use `scrapy.downloadermiddlewares.stats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.stats import * diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py deleted file mode 100644 index 893d5241c..000000000 --- a/scrapy/contrib/downloadermiddleware/useragent.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, " - "use `scrapy.downloadermiddlewares.useragent` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.useragent import * diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py deleted file mode 100644 index 12adaaddd..000000000 --- a/scrapy/contrib/exporter/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.exporter` is deprecated, " - "use `scrapy.exporters` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.exporters import * -from scrapy.exporters import PythonItemExporter diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py deleted file mode 100644 index 19651998a..000000000 --- a/scrapy/contrib/feedexport.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, " - "use `scrapy.extensions.feedexport` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.feedexport import * diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py deleted file mode 100644 index 196372fcb..000000000 --- a/scrapy/contrib/httpcache.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, " - "use `scrapy.extensions.httpcache` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.httpcache import * diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py deleted file mode 100644 index 976658df3..000000000 --- a/scrapy/contrib/linkextractors/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, " - "use `scrapy.linkextractors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors import * diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py deleted file mode 100644 index ff03da98f..000000000 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, " - "use `scrapy.linkextractors.htmlparser` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.htmlparser import * diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py deleted file mode 100644 index fc2b7de3c..000000000 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, " - "use `scrapy.linkextractors.lxmlhtml` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.lxmlhtml import * diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py deleted file mode 100644 index 97bda29c1..000000000 --- a/scrapy/contrib/linkextractors/regex.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, " - "use `scrapy.linkextractors.regex` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.regex import * diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py deleted file mode 100644 index a5a598208..000000000 --- a/scrapy/contrib/linkextractors/sgml.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, " - "use `scrapy.linkextractors.sgml` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.sgml import * diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py deleted file mode 100644 index 2b9453e18..000000000 --- a/scrapy/contrib/loader/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader` is deprecated, " - "use `scrapy.loader` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader import * diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py deleted file mode 100644 index a59b2b7b1..000000000 --- a/scrapy/contrib/loader/common.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, " - "use `scrapy.loader.common` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader.common import * diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py deleted file mode 100644 index da7e484a5..000000000 --- a/scrapy/contrib/loader/processor.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, " - "use `scrapy.loader.processors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader.processors import * diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py deleted file mode 100644 index 62bc9b860..000000000 --- a/scrapy/contrib/logstats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.logstats` is deprecated, " - "use `scrapy.extensions.logstats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.logstats import * diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py deleted file mode 100644 index 4f6e4760e..000000000 --- a/scrapy/contrib/memdebug.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, " - "use `scrapy.extensions.memdebug` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.memdebug import * diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py deleted file mode 100644 index e13bd78f3..000000000 --- a/scrapy/contrib/memusage.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.memusage` is deprecated, " - "use `scrapy.extensions.memusage` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.memusage import * diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py deleted file mode 100644 index aedf34a3f..000000000 --- a/scrapy/contrib/pipeline/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, " - "use `scrapy.pipelines` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines import * diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py deleted file mode 100644 index cd1238b5d..000000000 --- a/scrapy/contrib/pipeline/files.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, " - "use `scrapy.pipelines.files` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.files import * diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py deleted file mode 100644 index 4f5ce4c40..000000000 --- a/scrapy/contrib/pipeline/images.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, " - "use `scrapy.pipelines.images` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.images import * diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py deleted file mode 100644 index 4b4fea560..000000000 --- a/scrapy/contrib/pipeline/media.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, " - "use `scrapy.pipelines.media` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.media import * diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py deleted file mode 100644 index 718803148..000000000 --- a/scrapy/contrib/spidermiddleware/depth.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, " - "use `scrapy.spidermiddlewares.depth` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.depth import * diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py deleted file mode 100644 index e39fb3f56..000000000 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, " - "use `scrapy.spidermiddlewares.httperror` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.httperror import * diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py deleted file mode 100644 index a5ed9ea7e..000000000 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, " - "use `scrapy.spidermiddlewares.offsite` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.offsite import * diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py deleted file mode 100644 index fdf8d6659..000000000 --- a/scrapy/contrib/spidermiddleware/referer.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, " - "use `scrapy.spidermiddlewares.referer` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.referer import * diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py deleted file mode 100644 index 5e51add59..000000000 --- a/scrapy/contrib/spidermiddleware/urllength.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, " - "use `scrapy.spidermiddlewares.urllength` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.urllength import * diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py deleted file mode 100644 index 56780533b..000000000 --- a/scrapy/contrib/spiders/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders` is deprecated, " - "use `scrapy.spiders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders import * diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py deleted file mode 100644 index d20a8bb16..000000000 --- a/scrapy/contrib/spiders/crawl.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, " - "use `scrapy.spiders.crawl` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.crawl import * diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py deleted file mode 100644 index 5eea9a062..000000000 --- a/scrapy/contrib/spiders/feed.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, " - "use `scrapy.spiders.feed` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.feed import * diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py deleted file mode 100644 index 6d1ec0aa9..000000000 --- a/scrapy/contrib/spiders/init.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, " - "use `scrapy.spiders.init` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.init import * diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py deleted file mode 100644 index 2ad231fd8..000000000 --- a/scrapy/contrib/spiders/sitemap.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, " - "use `scrapy.spiders.sitemap` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.sitemap import * diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py deleted file mode 100644 index 06afc8bfc..000000000 --- a/scrapy/contrib/spiderstate.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, " - "use `scrapy.extensions.spiderstate` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.spiderstate import * diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py deleted file mode 100644 index f9c9a37f5..000000000 --- a/scrapy/contrib/statsmailer.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, " - "use `scrapy.extensions.statsmailer` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.statsmailer import * diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py deleted file mode 100644 index d5c234871..000000000 --- a/scrapy/contrib/throttle.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.throttle` is deprecated, " - "use `scrapy.extensions.throttle` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.throttle import * diff --git a/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/downloadermiddleware/__init__.py b/scrapy/contrib_exp/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py deleted file mode 100644 index 1f8490587..000000000 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.downloadermiddlewares.decompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py deleted file mode 100644 index c59f47bcc..000000000 --- a/scrapy/contrib_exp/iterators.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib_exp.iterators` is deprecated, use `scrapy.utils.iterators` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.utils.iterators import xmliter_lxml diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py deleted file mode 100644 index 232d96288..000000000 --- a/scrapy/dupefilter.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.dupefilter` is deprecated, " - "use `scrapy.dupefilters` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.dupefilters import * diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py deleted file mode 100644 index b744aff8e..000000000 --- a/scrapy/linkextractor.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.linkextractor` is deprecated, " - "use `scrapy.linkextractors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors import * diff --git a/scrapy/spider.py b/scrapy/spider.py deleted file mode 100644 index 56a5a0a0b..000000000 --- a/scrapy/spider.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.spider` is deprecated, " - "use `scrapy.spiders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders import * diff --git a/scrapy/squeue.py b/scrapy/squeue.py deleted file mode 100644 index a4a3f4238..000000000 --- a/scrapy/squeue.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.squeue` is deprecated, " - "use `scrapy.squeues` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.squeues import * diff --git a/scrapy/statscol.py b/scrapy/statscol.py deleted file mode 100644 index b4ddcce28..000000000 --- a/scrapy/statscol.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.statscol` is deprecated, " - "use `scrapy.statscollectors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.statscollectors import * diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py deleted file mode 100644 index e8c8eae39..000000000 --- a/scrapy/utils/decorator.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.utils.decorator` is deprecated, " - "use `scrapy.utils.decorators` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.utils.decorators import * diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f76161a68..8c72cc556 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -124,25 +124,7 @@ def _clspath(cls, forced=None): DEPRECATION_RULES = [ - ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'), - ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'), - ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'), - ('scrapy.contrib.exporter.', 'scrapy.exporters.'), - ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'), - ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'), - ('scrapy.contrib.loader.', 'scrapy.loader.'), - ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'), - ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'), - ('scrapy.contrib.spiders.', 'scrapy.spiders.'), - ('scrapy.contrib.', 'scrapy.extensions.'), - ('scrapy.command.', 'scrapy.commands.'), - ('scrapy.dupefilter.', 'scrapy.dupefilters.'), - ('scrapy.linkextractor.', 'scrapy.linkextractors.'), ('scrapy.telnet.', 'scrapy.extensions.telnet.'), - ('scrapy.spider.', 'scrapy.spiders.'), - ('scrapy.squeue.', 'scrapy.squeues.'), - ('scrapy.statscol.', 'scrapy.statscollectors.'), - ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'), ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'), ] From 36453348fad9babc96558ab10af9b2942eb5431e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:23:37 +0500 Subject: [PATCH 0072/1853] remove ancient modules kept only for error messages --- .coveragerc | 2 -- conftest.py | 2 -- scrapy/project.py | 17 ----------------- scrapy/stats.py | 8 -------- 4 files changed, 29 deletions(-) delete mode 100644 scrapy/project.py delete mode 100644 scrapy/stats.py diff --git a/.coveragerc b/.coveragerc index aeadccb25..1fde07e7e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,4 @@ omit = tests/* scrapy/xlib/* scrapy/conf.py - scrapy/stats.py - scrapy/project.py scrapy/log.py diff --git a/conftest.py b/conftest.py index c733db646..2d015f5e9 100644 --- a/conftest.py +++ b/conftest.py @@ -11,8 +11,6 @@ def _py_files(folder): collect_ignore = [ # deprecated or moved modules "scrapy/conf.py", - "scrapy/stats.py", - "scrapy/project.py", "scrapy/log.py", # not a test, but looks like a test diff --git a/scrapy/project.py b/scrapy/project.py deleted file mode 100644 index d8973a6c7..000000000 --- a/scrapy/project.py +++ /dev/null @@ -1,17 +0,0 @@ - -""" -Obsolete module, kept for giving a meaningful error message when trying to -import. -""" - -raise ImportError("""scrapy.project usage has become obsolete. - -If you want to get the Scrapy crawler from your extension, middleware or -pipeline implement the `from_crawler` class method (or look up for extending -components that have already done it, such as spiders). - -For example: - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler)""") diff --git a/scrapy/stats.py b/scrapy/stats.py deleted file mode 100644 index 710601430..000000000 --- a/scrapy/stats.py +++ /dev/null @@ -1,8 +0,0 @@ - -""" -Obsolete module, kept for giving a meaningful error message when trying to -import. -""" - -raise ImportError("scrapy.stats usage has become obsolete, use " - "`crawler.stats` attribute instead") From f531b66822491140740a6d86af2f3f11f0443d38 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:28:01 +0500 Subject: [PATCH 0073/1853] SpiderManager shim is removed --- scrapy/interfaces.py | 4 ---- scrapy/spidermanager.py | 7 ------- scrapy/utils/deprecate.py | 1 - 3 files changed, 12 deletions(-) delete mode 100644 scrapy/spidermanager.py diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index eb93c6f7e..89ad2b14f 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -16,7 +16,3 @@ class ISpiderLoader(Interface): def find_by_request(request): """Return the list of spiders names that can handle the given request""" - -# ISpiderManager is deprecated, don't use it! -# An alias is kept for backwards compatibility. -ISpiderManager = ISpiderLoader diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py deleted file mode 100644 index 220257bb1..000000000 --- a/scrapy/spidermanager.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Backwards compatibility shim. Use scrapy.spiderloader instead. -""" -from scrapy.spiderloader import SpiderLoader -from scrapy.utils.deprecate import create_deprecated_class - -SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 8c72cc556..2d3db431d 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -125,7 +125,6 @@ def _clspath(cls, forced=None): DEPRECATION_RULES = [ ('scrapy.telnet.', 'scrapy.extensions.telnet.'), - ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'), ] From 722e1afcdb337bf11652167f02435c81fc68ecfb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:21:19 +0300 Subject: [PATCH 0074/1853] Update ancient pytest on python 3 2.9 gives collection errors on python 3.7 due to PEP 479. --- tests/requirements-py3.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 51a25f5e5..8d9ce5231 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,6 @@ -pytest==2.9.2 +pytest==3.6.3 pytest-twisted -pytest-cov==2.2.1 +pytest-cov==2.5.1 testfixtures jmespath leveldb From 17e9914b8a12e5a96cc40016b74f301fb9835cbf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:09 +0300 Subject: [PATCH 0075/1853] Catch SyntaxError as well when importing manhole Also give a more detailed reason why telnet is not enabled (for the future). --- requirements-py3.txt | 2 +- scrapy/extensions/telnet.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index b941fd867..b38c4cc09 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index e78afa1fc..7cc8f823a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -12,7 +12,7 @@ try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True -except ImportError: +except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,7 +40,8 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured + raise NotConfigured('TelnetConsole not enabled: failed to import ' + 'required twisted modules.') self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From cf9399acc149cf5eafb2d00d310416ab2ba185e5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:56 +0300 Subject: [PATCH 0076/1853] Use python 3.7 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29f9f0065..f6ea670ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7-dev + - python: 3.7 env: TOXENV=py37 install: - | From 2773fe09e4b4fac51dbc06725f1fafb2e5d9a271 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:36:58 +0300 Subject: [PATCH 0077/1853] Make "docs" the last build, even though it still uses python3.6 for now --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6ea670ae..88c72b08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.6 - env: TOXENV=docs - python: 3.7 env: TOXENV=py37 + - python: 3.6 + env: TOXENV=docs install: - | if [ "$TOXENV" = "pypy" ]; then From f4f39057cbbfa4daf66f82061e57101b88d88d05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:46:45 +0300 Subject: [PATCH 0078/1853] Make csviter work on python 3.7 PEP 479 does not allow for StopIteration in generators. Instead, handle it explicitly, also use a for loop which looks simpler. --- scrapy/utils/iterators.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 73857b410..a12e14005 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): """ encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' - def _getrow(csv_r): - return [to_unicode(field, encoding) for field in next(csv_r)] + + def row_to_unicode(row_): + return [to_unicode(field, encoding) for field in row_] # Python 3 csv reader input object needs to return strings if six.PY3: @@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): csv_r = csv.reader(lines, **kwargs) if not headers: - headers = _getrow(csv_r) + try: + row = next(csv_r) + except StopIteration: + return + headers = row_to_unicode(row) - while True: - row = _getrow(csv_r) + for row in csv_r: + row = row_to_unicode(row) if len(row) != len(headers): logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, " "should be: %(csvheader)d)", From b3cd12dc48592fb8b1d4c6883315fe2b341ca5c2 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:53:40 +0300 Subject: [PATCH 0079/1853] Try to get python3.7 by using xenial base and sudo See https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-401756442 and https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-402045581 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 88c72b08e..4218d13bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 + dist: xenial + sudo: true - python: 3.6 env: TOXENV=docs install: From 92b504eae5379dadade2d78efba4e54b201cbd93 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 13:43:36 +0300 Subject: [PATCH 0080/1853] Fix telnet warnings in tests Disable telnet console if it's not available, else we'll get an extra warning about failure to enable it, and tests will fail. --- tests/test_crawler.py | 6 ++++-- tests/test_utils_log.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d3b80f460..6a8e11363 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import tempfile import warnings import unittest @@ -14,8 +13,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle +from scrapy.extensions import telnet + class BaseCrawlerTest(unittest.TestCase): @@ -100,6 +100,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file.name, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 45527b03b..742e04803 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -10,6 +10,7 @@ from twisted.python.failure import Failure from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter, LogCounterHandler, StreamLogger) from scrapy.utils.test import get_crawler +from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): @@ -65,10 +66,14 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): + settings = {'LOG_LEVEL': 'WARNING'} + if not telnet.TWISTED_CONCH_AVAILABLE: + # disable it to avoid the extra warning + settings['TELNETCONSOLE_ENABLED'] = False self.logger = logging.getLogger('test') self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'}) + self.crawler = get_crawler(settings_dict=settings) self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) From 4f6778aa7332aecd15f7672a0852d1f49596969b Mon Sep 17 00:00:00 2001 From: nyov Date: Mon, 9 Jul 2018 17:16:31 +0000 Subject: [PATCH 0081/1853] Remove deprecated CrawlerSettings class and Settings attributes --- docs/news.rst | 9 +++++ scrapy/settings/__init__.py | 48 -------------------------- tests/test_settings/__init__.py | 60 +-------------------------------- 3 files changed, 10 insertions(+), 107 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1b8d121a1..633e5c72f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,15 @@ Release notes ============= +Scrapy 1.6.0 (unreleased) +------------------------- + +Cleanups +~~~~~~~~ + +* Remove deprecated ``CrawlerSettings`` class. +* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. + Scrapy 1.5.0 (2017-12-29) ------------------------- diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 28446a372..7d6d20164 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -6,7 +6,6 @@ from collections import MutableMapping from importlib import import_module from pprint import pformat -from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning from . import default_settings @@ -405,30 +404,6 @@ class BaseSettings(MutableMapping): else: p.text(pformat(self.copy_to_dict())) - @property - def overrides(self): - warnings.warn("`Settings.overrides` attribute is deprecated and won't " - "be supported in Scrapy 0.26, use " - "`Settings.set(name, value, priority='cmdline')` instead", - category=ScrapyDeprecationWarning, stacklevel=2) - try: - o = self._overrides - except AttributeError: - self._overrides = o = _DictProxy(self, 'cmdline') - return o - - @property - def defaults(self): - warnings.warn("`Settings.defaults` attribute is deprecated and won't " - "be supported in Scrapy 0.26, use " - "`Settings.set(name, value, priority='default')` instead", - category=ScrapyDeprecationWarning, stacklevel=2) - try: - o = self._defaults - except AttributeError: - self._defaults = o = _DictProxy(self, 'default') - return o - class _DictProxy(MutableMapping): @@ -479,29 +454,6 @@ class Settings(BaseSettings): self.update(values, priority) -class CrawlerSettings(Settings): - - def __init__(self, settings_module=None, **kw): - self.settings_module = settings_module - Settings.__init__(self, **kw) - - def __getitem__(self, opt_name): - if opt_name in self.overrides: - return self.overrides[opt_name] - if self.settings_module and hasattr(self.settings_module, opt_name): - return getattr(self.settings_module, opt_name) - if opt_name in self.defaults: - return self.defaults[opt_name] - return Settings.__getitem__(self, opt_name) - - def __str__(self): - return "" % self.settings_module - -CrawlerSettings = create_deprecated_class( - 'CrawlerSettings', CrawlerSettings, - new_class_path='scrapy.settings.Settings') - - def iter_default_settings(): """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 863684075..1dbacbea3 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -3,8 +3,7 @@ import unittest import warnings from scrapy.settings import (BaseSettings, Settings, SettingsAttribute, - CrawlerSettings, SETTINGS_PRIORITIES, - get_settings_priority) + SETTINGS_PRIORITIES, get_settings_priority) from tests import mock from . import default_settings @@ -341,35 +340,6 @@ class BaseSettingsTest(unittest.TestCase): self.assertTrue(frozencopy.frozen) self.assertIsNot(frozencopy, self.settings) - def test_deprecated_attribute_overrides(self): - self.settings.set('BAR', 'fuz', priority='cmdline') - with warnings.catch_warnings(record=True) as w: - self.settings.overrides['BAR'] = 'foo' - self.assertIn("Settings.overrides", str(w[0].message)) - self.assertEqual(self.settings.get('BAR'), 'foo') - self.assertEqual(self.settings.overrides.get('BAR'), 'foo') - self.assertIn('BAR', self.settings.overrides) - - self.settings.overrides.update(BAR='bus') - self.assertEqual(self.settings.get('BAR'), 'bus') - self.assertEqual(self.settings.overrides.get('BAR'), 'bus') - - self.settings.overrides.setdefault('BAR', 'fez') - self.assertEqual(self.settings.get('BAR'), 'bus') - - self.settings.overrides.setdefault('FOO', 'fez') - self.assertEqual(self.settings.get('FOO'), 'fez') - self.assertEqual(self.settings.overrides.get('FOO'), 'fez') - - def test_deprecated_attribute_defaults(self): - self.settings.set('BAR', 'fuz', priority='default') - with warnings.catch_warnings(record=True) as w: - self.settings.defaults['BAR'] = 'foo' - self.assertIn("Settings.defaults", str(w[0].message)) - self.assertEqual(self.settings.get('BAR'), 'foo') - self.assertEqual(self.settings.defaults.get('BAR'), 'foo') - self.assertIn('BAR', self.settings.defaults) - class SettingsTest(unittest.TestCase): @@ -422,33 +392,5 @@ class SettingsTest(unittest.TestCase): self.assertEqual(mydict['key'], 'val') -class CrawlerSettingsTest(unittest.TestCase): - - def test_deprecated_crawlersettings(self): - def _get_settings(settings_dict=None): - settings_module = type('SettingsModuleMock', (object,), settings_dict or {}) - return CrawlerSettings(settings_module) - - with warnings.catch_warnings(record=True) as w: - settings = _get_settings() - self.assertIn("CrawlerSettings is deprecated", str(w[0].message)) - - # test_global_defaults - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180) - - # test_defaults - settings.defaults['DOWNLOAD_TIMEOUT'] = '99' - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99) - - # test_settings_module - settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'}) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3) - - # test_overrides - settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'}) - settings.overrides['DOWNLOAD_TIMEOUT'] = '15' - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15) - - if __name__ == "__main__": unittest.main() From 9428a4a3aa4c679c15eb2c606de7b49ad832ee6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 21:03:26 +0300 Subject: [PATCH 0082/1853] More visible telnet conch message Capture traceback when trying to import required twisted modules, print it in case telnet is enabled, and mention settings variable that can be used to supress the message. Thanks @kmike! --- scrapy/extensions/telnet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 7cc8f823a..3024ddfaa 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -6,6 +6,7 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging +import traceback from twisted.internet import protocol try: @@ -13,6 +14,7 @@ try: from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True except (ImportError, SyntaxError): + _TWISTED_CONCH_TRACEBACK = traceback.format_exc() TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,8 +42,9 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured('TelnetConsole not enabled: failed to import ' - 'required twisted modules.') + raise NotConfigured( + 'TELNETCONSOLE_ENABLED setting is True but required twisted ' + 'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK) self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From c86213317daf25aec04f3afc69c327a43033987f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 12 Jul 2018 02:10:24 +0500 Subject: [PATCH 0083/1853] 1.5.1 release notes --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 633e5c72f..01016e2e6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,6 +12,22 @@ Cleanups * Remove deprecated ``CrawlerSettings`` class. * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. + +Scrapy 1.5.1 (2018-07-12) +------------------------- + +This is a maintenance release with important bug fixes, but no new features: + +* ``O(N^2)`` gzip decompression issue which affected Python 3 and PyPy + is fixed (:issue:`3281`); +* skipping of TLS validation errors is improved (:issue:`3166`); +* Ctrl-C handling is fixed in Python 3.5+ (:issue:`3096`); +* testing fixes (:issue:`3092`, :issue:`3263`); +* documentation improvements (:issue:`3058`, :issue:`3059`, :issue:`3089`, + :issue:`3123`, :issue:`3127`, :issue:`3189`, :issue:`3224`, :issue:`3280`, + :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`, + :issue:`3294`). + Scrapy 1.5.0 (2017-12-29) ------------------------- From c61e8a617f1291bfcbe56d54a2f80f8fb79b7ddb Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Fri, 13 Jul 2018 11:55:16 -0700 Subject: [PATCH 0084/1853] [doc] update default RETRY_HTTP_CODES --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dfe4c13b4..8dbe249fa 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -882,7 +882,7 @@ precedence over the :setting:`RETRY_TIMES` setting. RETRY_HTTP_CODES ^^^^^^^^^^^^^^^^ -Default: ``[500, 502, 503, 504, 408]`` +Default: ``[500, 502, 503, 504, 522, 524, 408]`` Which HTTP response codes to retry. Other errors (DNS lookup issues, connections lost, etc) are always retried. From e7e18db179f2e45aa38a5bbdb0abba7d983cdce7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Jul 2018 14:04:35 -0300 Subject: [PATCH 0085/1853] Fix tests --- tests/test_spidermiddleware.py | 57 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2565ef7af..5622c3179 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -8,10 +8,7 @@ from twisted.internet import defer from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer - - -# TEST_URL = 'http://example.org' -TEST_URL = 'http://localhost:8998' +from tests.spiders import MockServerSpider class LogExceptionMiddleware: @@ -21,16 +18,18 @@ class LogExceptionMiddleware: # ================================================================================ -# recover from an exception on a spider callback +# (0) recover from an exception on a spider callback class RecoverySpider(Spider): name = 'RecoverySpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.RecoveryMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): yield {'test': 1} self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) @@ -61,10 +60,11 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request(TEST_URL, callback=self.parse, errback=self.errback) + yield Request(url=self.mockserver.url('/status?n=200'), + callback=self.parse, errback=self.errback) def parse(self, response): - return [{'test': 1}, {'test': 2}] + return {'from': 'callback'} def errback(self, failure): self.logger.warn('Got a Failure on the Request errback') @@ -80,13 +80,15 @@ class FailProcessSpiderInputMiddleware: # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): yield {'test': 1} yield {'test': 2} @@ -97,13 +99,15 @@ class GeneratorCallbackSpider(Spider): # (3) exceptions from a spider callback (not a generator) class NotAGeneratorCallbackSpider(Spider): name = 'NotAGeneratorCallbackSpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): return [{'test': 1}, {'test': 1/0}] @@ -123,20 +127,20 @@ class TestSpiderMiddleware(TestCase): def crawl_log(self, spider): crawler = get_crawler(spider) with LogCapture() as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) raise defer.returnValue(log) - # @defer.inlineCallbacks - # def test_recovery(self): - # """ - # Recover from an exception from a spider's callback. The final item count should be 3 - # (one from the spider before raising the exception, one from the middleware and one - # from the spider when processing the response that was enqueued from the middleware) - # """ - # log = yield self.crawl_log(RecoverySpider) - # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - # self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks + def test_recovery(self): + """ + (0) Recover from an exception in a spider callback. The final item count should be 2 + (one directly from the recovery middleware and one from the spider when processing + the request that was enqueued from the recovery middleware) + """ + log = yield self.crawl_log(RecoverySpider) + self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + self.assertIn("'item_scraped_count': 2", str(log)) @defer.inlineCallbacks def test_process_spider_input_errback(self): @@ -148,21 +152,24 @@ class TestSpiderMiddleware(TestCase): self.assertNotIn("Middleware: IndexError exception caught", str(log1)) self.assertIn("Middleware: will raise IndexError", str(log1)) self.assertIn("Got a Failure on the Request errback", str(log1)) + self.assertIn("{'from': 'errback'}", str(log1)) + self.assertNotIn("{'from': 'callback'}", str(log1)) + self.assertIn("'item_scraped_count': 1", str(log1)) @defer.inlineCallbacks def test_generator_callback(self): """ - (2) An exception from a spider's callback should + (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + self.assertNotIn("item_scraped_count", str(log2)) @defer.inlineCallbacks def test_not_a_generator_callback(self): """ - (3) An exception from a spider's callback should + (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain """ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) From 0c579b5276f502832f375be316094b4244cf87c5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 14 Jul 2018 19:58:42 -0300 Subject: [PATCH 0086/1853] Untested experiment --- scrapy/core/spidermw.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9dd8c91e..a9aeb6dcc 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from itertools import chain + import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput @@ -10,9 +12,28 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list + def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') + +class MutableChain: + def __init__(self, *args): + self.data = chain(*args) + + def extend(self, iterable): + self.data = chain(self.data, iterable) + + def __iter__(self): + return self.data.__iter__() + + def __next__(self): # py3 + return self.data.__next__() + + def next(self): # py2 + return self.data.next() + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -68,28 +89,32 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def process_spider_output(result, index): - def wrapper(result_iterable): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + def evaluate_result(result_iterable, index): try: for r in result_iterable: yield r except Exception as ex: - # process the exception with the method from the next middleware exception_result = process_spider_exception(Failure(ex), index) if exception_result is None or isinstance(exception_result, Failure): raise - for output in exception_result: - yield output + recovered.extend(exception_result) + for i, method in enumerate(self.methods['process_spider_output']): if i < index or method is None: continue result = method(response=response, result=result, spider=spider) index += 1 if _isiterable(result): - result = wrapper(result) + result = evaluate_result(result, index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) - return result + + return chain(result, recovered) dfd = mustbe_deferred(process_spider_input, response) dfd.addErrback(process_spider_exception, index=0) From c5fa0ae6bc536d6bc5370c6f3b634c33848971e5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 14 Jul 2018 19:58:42 -0300 Subject: [PATCH 0087/1853] Untested experiment --- scrapy/core/spidermw.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9dd8c91e..8ee42c2cf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from itertools import chain + import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput @@ -10,9 +12,31 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list + def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') + +class MutableChain: + """ + Thin wrapper around itertools.chain, allowing to add iterables "in-place" + """ + def __init__(self, *args): + self.data = chain(*args) + + def extend(self, *iterables): + self.data = chain(self.data, *iterables) + + def __iter__(self): + return self.data.__iter__() + + def __next__(self): # py3 + return self.data.__next__() + + def next(self): # py2 + return self.data.next() + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -68,28 +92,32 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def process_spider_output(result, index): - def wrapper(result_iterable): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + def evaluate_result(result_iterable, index): try: for r in result_iterable: yield r except Exception as ex: - # process the exception with the method from the next middleware exception_result = process_spider_exception(Failure(ex), index) if exception_result is None or isinstance(exception_result, Failure): raise - for output in exception_result: - yield output + recovered.extend(exception_result) + for i, method in enumerate(self.methods['process_spider_output']): if i < index or method is None: continue result = method(response=response, result=result, spider=spider) index += 1 if _isiterable(result): - result = wrapper(result) + result = evaluate_result(result, index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) - return result + + return chain(result, recovered) dfd = mustbe_deferred(process_spider_input, response) dfd.addErrback(process_spider_exception, index=0) From cff9e8762512033da181293bab379b485aeffa66 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 15 Jul 2018 16:21:08 -0300 Subject: [PATCH 0088/1853] Fix tests --- tests/test_spidermiddleware.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 5622c3179..c33eb28ca 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -68,6 +68,7 @@ class ProcessSpiderInputSpider(Spider): def errback(self, failure): self.logger.warn('Got a Failure on the Request errback') + return {'from': 'errback'} class FailProcessSpiderInputMiddleware: @@ -133,14 +134,15 @@ class TestSpiderMiddleware(TestCase): @defer.inlineCallbacks def test_recovery(self): """ - (0) Recover from an exception in a spider callback. The final item count should be 2 - (one directly from the recovery middleware and one from the spider when processing - the request that was enqueued from the recovery middleware) + (0) Recover from an exception in a spider callback. The final item count should be 3 + (one yielded from the callback method before the exception is raised, one directly + from the recovery middleware and one from the spider when processing the request that + was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - self.assertIn("'item_scraped_count': 2", str(log)) + self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks def test_process_spider_input_errback(self): @@ -164,7 +166,7 @@ class TestSpiderMiddleware(TestCase): """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertNotIn("item_scraped_count", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) @defer.inlineCallbacks def test_not_a_generator_callback(self): From 60c2ef86f0c40d17219d3e3320072fb5b1ded412 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 15 Jul 2018 16:47:55 -0300 Subject: [PATCH 0089/1853] Revert "Default values for OffsiteMiddleware" This reverts commit ba294351381c0dd81476603246d2cea6c31486be. --- scrapy/spidermiddlewares/offsite.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 3b7f194e4..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -19,9 +19,6 @@ class OffsiteMiddleware(object): def __init__(self, stats): self.stats = stats - # default values - self.host_regex = re.compile('') # allow all by default - self.domains_seen = set() @classmethod def from_crawler(cls, crawler): @@ -55,7 +52,7 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return + return re.compile('') # allow all by default url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): @@ -65,9 +62,8 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - host_regex = self.get_host_regex(spider) - if host_regex: - self.host_regex = host_regex + self.host_regex = self.get_host_regex(spider) + self.domains_seen = set() class URLWarning(Warning): From b8e8922d5436247f7be66c40e1a16a0acab7986e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 15 Jul 2018 17:50:55 -0300 Subject: [PATCH 0090/1853] Simplify stuff. Add more tests. --- scrapy/core/spidermw.py | 10 +++--- tests/test_spidermiddleware.py | 63 ++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 8ee42c2cf..c733402b9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -75,8 +75,8 @@ class SpiderMiddlewareManager(MiddlewareManager): # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for i, method in enumerate(self.methods['process_spider_exception']): - if i < index or method is None: + for method in self.methods['process_spider_exception'][index:]: + if method is None: continue result = method(response=response, exception=exception, spider=spider) index += 1 @@ -101,13 +101,13 @@ class SpiderMiddlewareManager(MiddlewareManager): for r in result_iterable: yield r except Exception as ex: - exception_result = process_spider_exception(Failure(ex), index) + exception_result = process_spider_exception(Failure(ex), index+1) if exception_result is None or isinstance(exception_result, Failure): raise recovered.extend(exception_result) - for i, method in enumerate(self.methods['process_spider_output']): - if i < index or method is None: + for method in self.methods['process_spider_output'][index:]: + if method is None: continue result = method(response=response, result=result, spider=spider) index += 1 diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index c33eb28ca..645d95059 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -113,6 +113,49 @@ class NotAGeneratorCallbackSpider(Spider): return [{'test': 1}, {'test': 1/0}] +# ================================================================================ +# (4) exceptions from a middleware process_spider_output method (generator) +class GeneratorOutputChainSpider(Spider): + name = 'GeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.GeneratorFailOutputChainMiddleware': 10, + __name__ + '.GeneratorRecoverOutputChainMiddleware': 5, + }, + } + + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + + def parse(self, response): + yield {'processed': ['parse']} + + +class GeneratorFailOutputChainMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + yield r + raise LookupError() + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + yield {'processed': [method]} + + +class GeneratorRecoverOutputChainMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + yield r + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + yield {'processed': [method]} + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod @@ -162,7 +205,8 @@ class TestSpiderMiddleware(TestCase): def test_generator_callback(self): """ (2) An exception from a spider callback (returning a generator) should - be caught by the process_spider_exception chain + be caught by the process_spider_exception chain. Items yielded before the + exception is raised should be processed normally. """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) @@ -172,8 +216,23 @@ class TestSpiderMiddleware(TestCase): def test_not_a_generator_callback(self): """ (3) An exception from a spider callback (returning a list) should - be caught by the process_spider_exception chain + be caught by the process_spider_exception chain. No items should be processed. """ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) self.assertNotIn("item_scraped_count", str(log3)) + + @defer.inlineCallbacks + def test_generator_output_chain(self): + """ + (4) An exception from a middleware's process_spider_output method should be sent + to the process_spider_exception method from the next middleware in the chain. + The final item count should be 2 (one from the spider callback and one from the + process_spider_exception chain) + """ + log4 = yield self.crawl_log(GeneratorOutputChainSpider) + self.assertIn("'item_scraped_count': 2", str(log4)) + self.assertIn("GeneratorRecoverOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertNotIn("GeneratorFailOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn("{'processed': ['parse', 'GeneratorFailOutputChainMiddleware.process_spider_output', 'GeneratorRecoverOutputChainMiddleware.process_spider_output']}", str(log4)) + self.assertIn("{'processed': ['GeneratorRecoverOutputChainMiddleware.process_spider_exception']}", str(log4)) From 56e92d90fda3e812aca270327e513391591a10cc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 17 Jul 2018 15:15:38 -0300 Subject: [PATCH 0091/1853] Update tests --- tests/test_spidermiddleware.py | 54 ++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 645d95059..9bb7f62fd 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -119,8 +119,10 @@ class GeneratorOutputChainSpider(Spider): name = 'GeneratorOutputChainSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.GeneratorFailOutputChainMiddleware': 10, - __name__ + '.GeneratorRecoverOutputChainMiddleware': 5, + __name__ + '.GeneratorFailMiddleware': 10, + __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8, + __name__ + '.GeneratorRecoverMiddleware': 5, + __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3, }, } @@ -128,10 +130,23 @@ class GeneratorOutputChainSpider(Spider): yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - yield {'processed': ['parse']} + yield {'processed': ['parse-first-item']} + yield {'processed': ['parse-second-item']} -class GeneratorFailOutputChainMiddleware: +class _GeneratorDoNothingMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + yield r + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return None + + +class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) @@ -144,7 +159,11 @@ class GeneratorFailOutputChainMiddleware: yield {'processed': [method]} -class GeneratorRecoverOutputChainMiddleware: +class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): + pass + + +class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) @@ -155,6 +174,9 @@ class GeneratorRecoverOutputChainMiddleware: logging.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} +class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): + pass + # ================================================================================ class TestSpiderMiddleware(TestCase): @@ -227,12 +249,26 @@ class TestSpiderMiddleware(TestCase): """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. + The result of the recovery by the process_spider_exception method should be handled + by the process_spider_output method from the next middleware. The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn("GeneratorRecoverOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorFailOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertIn("{'processed': ['parse', 'GeneratorFailOutputChainMiddleware.process_spider_output', 'GeneratorRecoverOutputChainMiddleware.process_spider_output']}", str(log4)) - self.assertIn("{'processed': ['GeneratorRecoverOutputChainMiddleware.process_spider_exception']}", str(log4)) + self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4)) + item_from_callback = {'processed': [ + 'parse-first-item', + 'GeneratorFailMiddleware.process_spider_output', + 'GeneratorDoNothingAfterFailureMiddleware.process_spider_output', + 'GeneratorRecoverMiddleware.process_spider_output', + 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + item_recovered = {'processed': [ + 'GeneratorRecoverMiddleware.process_spider_exception', + 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + self.assertIn(str(item_from_callback), str(log4)) + self.assertIn(str(item_recovered), str(log4)) + self.assertNotIn('parse-second-item', str(log4)) From 610f589662ca9c5929527e15cad2c347c8d5d335 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 17 Jul 2018 19:13:03 -0300 Subject: [PATCH 0092/1853] Add callback and errback in the same step --- scrapy/core/spidermw.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c733402b9..da51bc974 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -70,7 +70,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure, index): + def process_spider_exception(_failure, index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -91,7 +91,7 @@ class SpiderMiddlewareManager(MiddlewareManager): .format(fname(method), type(result))) return _failure - def process_spider_output(result, index): + def process_spider_output(result, index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -120,8 +120,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return chain(result, recovered) dfd = mustbe_deferred(process_spider_input, response) - dfd.addErrback(process_spider_exception, index=0) - dfd.addCallback(process_spider_output, index=0) + dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception) return dfd def process_start_requests(self, start_requests, spider): From a3af0bfd56770aab0a056ae6e29efffa8b7d88c4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Jul 2018 15:15:55 -0300 Subject: [PATCH 0093/1853] More tests --- scrapy/core/spidermw.py | 21 +++++--- tests/test_spidermiddleware.py | 99 ++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index da51bc974..96488806d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -64,8 +64,8 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - raise _InvalidOutput('Middleware {} must return None or raise ' \ - 'an exception, got {}'.format(fname(method), type(result))) + raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \ + .format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) @@ -78,8 +78,8 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods['process_spider_exception'][index:]: if method is None: continue - result = method(response=response, exception=exception, spider=spider) index += 1 + result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned @@ -96,9 +96,9 @@ class SpiderMiddlewareManager(MiddlewareManager): # chain, they went through it already from the process_spider_exception method recovered = MutableChain() - def evaluate_result(result_iterable, index): + def evaluate_iterable(iterable, index): try: - for r in result_iterable: + for r in iterable: yield r except Exception as ex: exception_result = process_spider_exception(Failure(ex), index+1) @@ -109,10 +109,17 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods['process_spider_output'][index:]: if method is None: continue - result = method(response=response, result=result, spider=spider) index += 1 + # the following might fail directly if the output value is not a generator + try: + result = method(response=response, result=result, spider=spider) + except Exception as ex: + exception_result = process_spider_exception(Failure(ex), index+1) + if exception_result is None or isinstance(exception_result, Failure): + raise + return exception_result if _isiterable(result): - result = evaluate_result(result, index) + result = evaluate_iterable(result, index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9bb7f62fd..2f431ddc7 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -98,8 +98,8 @@ class GeneratorCallbackSpider(Spider): # ================================================================================ # (3) exceptions from a spider callback (not a generator) -class NotAGeneratorCallbackSpider(Spider): - name = 'NotAGeneratorCallbackSpider' +class NotGeneratorCallbackSpider(Spider): + name = 'NotGeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, @@ -178,6 +178,76 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass +# ================================================================================ +# (5) exceptions from a middleware process_spider_output method (not generator) +class NotGeneratorOutputChainSpider(Spider): + name = 'NotGeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.NotGeneratorFailMiddleware': 10, + __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8, + __name__ + '.NotGeneratorRecoverMiddleware': 5, + __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3, + }, + } + + def start_requests(self): + return [Request(self.mockserver.url('/status?n=200'))] + + def parse(self, response): + return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + + +class _NotGeneratorDoNothingMiddleware: + def process_spider_output(self, response, result, spider): + out = [] + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + out.append(r) + return out + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return None + + +class NotGeneratorFailMiddleware: + def process_spider_output(self, response, result, spider): + out = [] + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + out.append(r) + raise ReferenceError() + return out + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return [{'processed': [method]}] + + +class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware): + pass + + +class NotGeneratorRecoverMiddleware: + def process_spider_output(self, response, result, spider): + out = [] + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + out.append(r) + return out + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return [{'processed': [method]}] + +class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): + pass + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod @@ -240,7 +310,7 @@ class TestSpiderMiddleware(TestCase): (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) + log3 = yield self.crawl_log(NotGeneratorCallbackSpider) self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) self.assertNotIn("item_scraped_count", str(log3)) @@ -272,3 +342,26 @@ class TestSpiderMiddleware(TestCase): self.assertIn(str(item_from_callback), str(log4)) self.assertIn(str(item_recovered), str(log4)) self.assertNotIn('parse-second-item', str(log4)) + + @defer.inlineCallbacks + def test_not_a_generator_output_chain(self): + """ + (5) An exception from a middleware's process_spider_output method should be sent + to the process_spider_exception method from the next middleware in the chain. + The result of the recovery by the process_spider_exception method should be handled + by the process_spider_output method from the next middleware. + The final item count should be 1 (from the process_spider_exception chain, the items + from the spider callback are lost) + """ + log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) + self.assertIn("'item_scraped_count': 1", str(log5)) + self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + item_recovered = {'processed': [ + 'NotGeneratorRecoverMiddleware.process_spider_exception', + 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + self.assertIn(str(item_recovered), str(log5)) + self.assertNotIn('parse-first-item', str(log5)) + self.assertNotIn('parse-second-item', str(log5)) From 6329441c826bec97aeec82d3e7ec0bcd91c60a47 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Jul 2018 16:59:24 -0300 Subject: [PATCH 0094/1853] ModuleNotFoundError was added in py3.6 --- tests/test_spidermiddleware.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2f431ddc7..0451dfd27 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -34,7 +34,7 @@ class RecoverySpider(Spider): yield {'test': 1} self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) if not response.meta.get('dont_fail'): - raise ModuleNotFoundError() + raise TabError() class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): @@ -275,8 +275,8 @@ class TestSpiderMiddleware(TestCase): was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + self.assertIn("Middleware: TabError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks From 71a1406c99e7d4cced0693389e537c98a38104aa Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Jul 2018 17:40:30 -0300 Subject: [PATCH 0095/1853] Logging changes --- tests/test_spidermiddleware.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 0451dfd27..0f5646a72 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,4 @@ -import logging - from testfixtures import LogCapture from twisted.trial.unittest import TestCase from twisted.internet import defer @@ -13,7 +11,7 @@ from tests.spiders import MockServerSpider class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) return None @@ -32,13 +30,13 @@ class RecoverySpider(Spider): def parse(self, response): yield {'test': 1} - self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) + self.logger.info('DONT_FAIL: %s', response.meta.get('dont_fail')) if not response.meta.get('dont_fail'): raise TabError() class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) return [ {'from': 'process_spider_exception'}, Request(response.url, meta={'dont_fail': True}, dont_filter=True), @@ -67,13 +65,13 @@ class ProcessSpiderInputSpider(Spider): return {'from': 'callback'} def errback(self, failure): - self.logger.warn('Got a Failure on the Request errback') + self.logger.info('Got a Failure on the Request errback') return {'from': 'errback'} class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): - logging.warn('Middleware: will raise IndexError') + spider.logger.info('Middleware: will raise IndexError') raise IndexError() @@ -142,7 +140,7 @@ class _GeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -155,7 +153,7 @@ class GeneratorFailMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -171,7 +169,7 @@ class GeneratorRecoverMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): @@ -208,7 +206,7 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -223,7 +221,7 @@ class NotGeneratorFailMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] @@ -241,7 +239,7 @@ class NotGeneratorRecoverMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): From 20defa2e16628b4b432a1cc44ad37182dfc764ee Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Jul 2018 10:31:06 -0300 Subject: [PATCH 0096/1853] Better handling of method indexes --- scrapy/core/spidermw.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 96488806d..8607ed620 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -70,20 +70,19 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure, index=0): + def process_spider_exception(_failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for method in self.methods['process_spider_exception'][index:]: + for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index): if method is None: continue - index += 1 result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return process_spider_output(result, index) + return process_spider_output(result, method_index+1) elif result is None: continue else: @@ -91,7 +90,7 @@ class SpiderMiddlewareManager(MiddlewareManager): .format(fname(method), type(result))) return _failure - def process_spider_output(result, index=0): + def process_spider_output(result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -106,20 +105,19 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recovered.extend(exception_result) - for method in self.methods['process_spider_output'][index:]: + for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index): if method is None: continue - index += 1 # the following might fail directly if the output value is not a generator try: result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), index+1) + exception_result = process_spider_exception(Failure(ex), method_index+1) if exception_result is None or isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): - result = evaluate_iterable(result, index) + result = evaluate_iterable(result, method_index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) From 6a38fc39f8fd1344bc41fcf50fe0e0af27ec74c4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Jul 2018 11:56:23 -0300 Subject: [PATCH 0097/1853] Include flags when copying requests --- scrapy/http/request/__init__.py | 2 +- tests/test_http_request.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 13a92ffa0..cd4360483 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -91,7 +91,7 @@ class Request(object_ref): """Create a new Request with the same attributes except for those given new values. """ - for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', + for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a042f03b6..fc89229c6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -174,7 +174,8 @@ class RequestTest(unittest.TestCase): def somecallback(): pass - r1 = self.request_class("http://www.example.com", callback=somecallback, errback=somecallback) + r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], + callback=somecallback, errback=somecallback) r1.meta['foo'] = 'bar' r2 = r1.copy() @@ -184,6 +185,10 @@ class RequestTest(unittest.TestCase): assert r2.callback is r1.callback assert r2.errback is r2.errback + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + self.assertEqual(r1.flags, r2.flags) + # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" self.assertEqual(r1.meta, r2.meta) From 7020c3e4523ef445f0d279cdb85fc7570d9e2c2e Mon Sep 17 00:00:00 2001 From: Andrei Korigodski Date: Fri, 20 Jul 2018 14:46:57 +0300 Subject: [PATCH 0098/1853] Doc: update copyright notice The years are updated. The hyphen is replaced with an en dash. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 594740f39..a54a6bbe9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,7 +45,7 @@ master_doc = 'index' # General information about the project. project = u'Scrapy' -copyright = u'2008-2016, Scrapy developers' +copyright = u'2008–2018, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 98d74d1083be6afa7553a5950f89fc5ac446272f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 20 Jul 2018 12:08:49 -0300 Subject: [PATCH 0099/1853] Requested changes --- scrapy/extensions/feedexport.py | 29 +++++++++++++++-------------- scrapy/utils/misc.py | 1 + tests/test_feedexport.py | 4 +++- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 067887d94..7c7db387e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -96,21 +96,22 @@ class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None): # BEGIN Backwards compatibility for initialising without keys (and # without using from_crawler) - from scrapy.conf import settings no_defaults = access_key is None and secret_key is None - if no_defaults and ('AWS_ACCESS_KEY_ID' in settings or - 'AWS_SECRET_ACCESS_KEY' in settings): - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn( - "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " - "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] + if no_defaults: + from scrapy.conf import settings + if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn( + "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " + "without AWS keys is deprecated. Please supply credentials or " + "use the `from_crawler()` constructor.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + access_key = settings['AWS_ACCESS_KEY_ID'] + secret_key = settings['AWS_SECRET_ACCESS_KEY'] + # END Backwards compatibility u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8eb1aabb5..5ccfdcd72 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -118,6 +118,7 @@ def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False + def create_instance(objcls, settings, crawler, *args, **kwargs): """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructors, if available. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eeb1bc2a4..380ed971b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -173,7 +173,9 @@ class S3FeedStorageTest(unittest.TestCase): uri = os.environ.get('S3_TEST_FILE_URI') if not uri: raise unittest.SkipTest("No S3 URI available for testing") - storage = S3FeedStorage(uri, Settings()) + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + storage = S3FeedStorage(uri, access_key, secret_key) verifyObject(IFeedStorage, storage) file = storage.open(scrapy.Spider("default")) expected_content = b"content: \xe2\x98\x83" From 784eed113021a1a787787a354add242e7abbf6f9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 20 Jul 2018 19:08:46 -0300 Subject: [PATCH 0100/1853] Improve test coverage (downloader middleware) --- scrapy/core/downloader/middleware.py | 6 +-- tests/test_downloadermiddleware.py | 64 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index cf0c1f869..2fa277e7d 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -50,8 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): defer.returnValue(response) for method in self.methods['process_response']: - response = yield method(request=request, response=response, - spider=spider) + response = yield method(request=request, response=response, spider=spider) if not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ (six.get_method_self(method).__class__.__name__, type(response))) @@ -63,8 +62,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(_failure): exception = _failure.value for method in self.methods['process_exception']: - response = yield method(request=request, exception=exception, - spider=spider) + response = yield method(request=request, exception=exception, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ (six.get_method_self(method).__class__.__name__, type(response))) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index fb51392b2..0f420b70d 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -3,6 +3,7 @@ from twisted.python.failure import Failure from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.exceptions import _InvalidOutput from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes @@ -115,3 +116,66 @@ class ResponseFromProcessRequestTest(ManagerTestCase): self.assertIs(results[0], resp) self.assertFalse(download_func.called) + + +class ProcessRequestInvalidOutput(ManagerTestCase): + """Invalid return value for process_request method should raise an exception""" + + def test_invalid_process_request(self): + req = Request('http://example.com/index.html') + resp = Response('http://example.com/index.html') + + class InvalidProcessRequestMiddleware: + def process_request(self, request, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessRequestMiddleware()) + download_func = mock.MagicMock() + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self.assertIsInstance(results[0], Failure) + self.assertIsInstance(results[0].value, _InvalidOutput) + + +class ProcessResponseInvalidOutput(ManagerTestCase): + """Invalid return value for process_response method should raise an exception""" + + def test_invalid_process_response(self): + req = Request('http://example.com/index.html') + resp = Response('http://example.com/index.html') + + class InvalidProcessResponseMiddleware: + def process_response(self, request, response, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessResponseMiddleware()) + download_func = mock.MagicMock() + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self.assertIsInstance(results[0], Failure) + self.assertIsInstance(results[0].value, _InvalidOutput) + + +class ProcessExceptionInvalidOutput(ManagerTestCase): + """Invalid return value for process_exception method should raise an exception""" + + def test_invalid_process_exception(self): + req = Request('http://example.com/index.html') + resp = Response('http://example.com/index.html') + + class InvalidProcessExceptionMiddleware: + def process_request(self, request, spider): + raise Exception() + + def process_exception(self, request, exception, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) + download_func = mock.MagicMock() + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self.assertIsInstance(results[0], Failure) + self.assertIsInstance(results[0].value, _InvalidOutput) From 48866457b309ebb5edfe2dac8966af1c825b9d65 Mon Sep 17 00:00:00 2001 From: Malcolm Granado Ho Yong Liang Date: Wed, 25 Jul 2018 14:38:37 +0800 Subject: [PATCH 0101/1853] make amendments to grammer --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 6615840f7..2369c3436 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -12,7 +12,7 @@ Contributing to Scrapy There are many ways to contribute to Scrapy. Here are some of them: * Blog about Scrapy. Tell the world how you're using Scrapy. This will help - newcomers with more examples and the Scrapy project to increase its + newcomers with more examples and will help the Scrapy project to increase its visibility. * Report bugs and request features in the `issue tracker`_, trying to follow @@ -39,7 +39,7 @@ Reporting bugs trusted Scrapy developers, and its archives are not public. Well-written bug reports are very helpful, so keep in mind the following -guidelines when reporting a new bug. +guidelines when you're going to report a new bug. * check the :ref:`FAQ ` first to see if your issue is addressed in a well-known question From 782f866572d8bf8b1673ea962ae57d7631d1d9de Mon Sep 17 00:00:00 2001 From: CCInCharge Date: Thu, 7 Jun 2018 16:39:48 -0700 Subject: [PATCH 0102/1853] Fix #3247: Allow scrapy.FormRequest.from_response method to handle duplicate keys --- scrapy/http/request/form.py | 11 ++++++++--- tests/test_http_request.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 95b38e990..c2413b431 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath): def _get_inputs(form, formdata, dont_click, clickdata, response): try: - formdata = dict(formdata or ()) + formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') + if not formdata: + formdata = () inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' @@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): "re": "http://exslt.org/regular-expressions"}) values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) - if k and k not in formdata] + if k and k not in formdata_keys] if not dont_click: clickable = _get_clickable(clickdata, form) if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - values.extend((k, v) for k, v in formdata.items() if v is not None) + if isinstance(formdata, dict): + formdata = formdata.items() + + values.extend((k, v) for k, v in formdata if v is not None) return values diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a042f03b6..18fc6413c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -401,6 +401,29 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[u'test2'], [u'xxx µ']) self.assertEqual(fs[u'six'], [u'seven']) + def test_from_response_duplicate_form_key(self): + response = _buildresponse( + '
', + url='http://www.example.com') + req = self.request_class.from_response(response, + method='GET', + formdata=(('foo', 'bar'), ('foo', 'baz'))) + self.assertEqual(urlparse(req.url).hostname, 'www.example.com') + self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') + + def test_from_response_override_duplicate_form_key(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response( + response, + formdata=(('two', '2'), ('two', '4'))) + fs = _qs(req) + self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b'two'], [b'2', b'4']) + def test_from_response_extra_headers(self): response = _buildresponse( """
From 701cd2ff9d4be34fcf63a7410d09f425410635f9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Oct 2017 09:42:34 -0300 Subject: [PATCH 0103/1853] Add from_crawler support to dupefilters --- scrapy/core/scheduler.py | 7 ++++++- tests/test_dupefilters.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a54b4daf0..faed27fd1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -26,7 +26,12 @@ class Scheduler(object): def from_crawler(cls, crawler): settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = dupefilter_cls.from_settings(settings) + if hasattr(dupefilter_cls, 'from_crawler'): + dupefilter = dupefilter_cls.from_crawler(crawler) + elif hasattr(dupefilter_cls, 'from_settings'): + dupefilter = dupefilter_cls.from_settings(crawler.settings) + else: + dupefilter = dupefilter_cls() pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 2d1a4bfff..81524fddd 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -6,10 +6,31 @@ import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.utils.python import to_bytes +from scrapy.utils.job import job_dir +from scrapy.utils.test import get_crawler class RFPDupeFilterTest(unittest.TestCase): + def test_dupefilter_from_crawler(self): + + class FromCrawlerRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_crawler(cls, crawler): + debug = crawler.settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(crawler.settings), debug) + df.user_agent = crawler.settings.get('USER_AGENT') + return df + + crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'}) + dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler) + + self.assertTrue(dupefilter.debug) + self.assertEqual(dupefilter.user_agent, 'test ua') + + dupefilter.close('finished') + def test_filter(self): dupefilter = RFPDupeFilter() dupefilter.open() From d306fe30ac08401e74fd20cf90e5164a4125d8e1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 30 Dec 2017 22:49:22 -0300 Subject: [PATCH 0104/1853] Test dupefilter creation by the Scheduler --- tests/test_dupefilters.py | 52 ++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 81524fddd..990b5141c 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -5,31 +5,49 @@ import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request +from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler +class FromCrawlerRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_crawler(cls, crawler): + debug = crawler.settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(crawler.settings), debug) + df.method = crawler.settings.get('METHOD') + return df + + +class FromSettingsRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_settings(cls, settings): + debug = settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(settings), debug) + df.method = settings.get('METHOD') + return df + + class RFPDupeFilterTest(unittest.TestCase): - def test_dupefilter_from_crawler(self): + def test_from_crawler_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_crawler', + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'from_crawler') - class FromCrawlerRFPDupeFilter(RFPDupeFilter): - - @classmethod - def from_crawler(cls, crawler): - debug = crawler.settings.getbool('DUPEFILTER_DEBUG') - df = cls(job_dir(crawler.settings), debug) - df.user_agent = crawler.settings.get('USER_AGENT') - return df - - crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'}) - dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler) - - self.assertTrue(dupefilter.debug) - self.assertEqual(dupefilter.user_agent, 'test ua') - - dupefilter.close('finished') + def test_from_settings_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_settings', + 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'from_settings') def test_filter(self): dupefilter = RFPDupeFilter() From 0089a4ab31d1764dd38c30c4448a4c62efd9b9c3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 23 Mar 2018 13:19:31 -0300 Subject: [PATCH 0105/1853] Add test for direct creation of dupefilter (no from_crawler/from_settings) --- tests/test_dupefilters.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 990b5141c..02a01fc94 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -17,7 +17,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): def from_crawler(cls, crawler): debug = crawler.settings.getbool('DUPEFILTER_DEBUG') df = cls(job_dir(crawler.settings), debug) - df.method = crawler.settings.get('METHOD') + df.method = 'from_crawler' return df @@ -27,28 +27,40 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): def from_settings(cls, settings): debug = settings.getbool('DUPEFILTER_DEBUG') df = cls(job_dir(settings), debug) - df.method = settings.get('METHOD') + df.method = 'from_settings' return df +class DirectRFPDupeFilter(RFPDupeFilter): + method = 'n/a' + + class RFPDupeFilterTest(unittest.TestCase): - def test_from_crawler_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_crawler', + def test_df_from_crawler_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_crawler') - def test_from_settings_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_settings', + def test_df_from_settings_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') + def test_df_direct_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.DirectRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'n/a') + def test_filter(self): dupefilter = RFPDupeFilter() dupefilter.open() From 9e14f8c7e4141fd5216efc378c9d003ea981a4d5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 23 Mar 2018 21:19:57 -0300 Subject: [PATCH 0106/1853] Fix test for dupefilter --- tests/test_dupefilters.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 02a01fc94..db69597a2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -31,7 +31,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): return df -class DirectRFPDupeFilter(RFPDupeFilter): +class DirectDupeFilter(object): method = 'n/a' @@ -54,11 +54,9 @@ class RFPDupeFilterTest(unittest.TestCase): self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.DirectRFPDupeFilter'} + settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'n/a') def test_filter(self): From 999341b60bd5289ca97c8187d0ff380a8555ba5e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 20 Jul 2018 22:17:55 -0300 Subject: [PATCH 0107/1853] Simplify dupefilter creation --- scrapy/core/scheduler.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index faed27fd1..eb790a67e 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import logging from os.path import join, exists from scrapy.utils.reqser import request_to_dict, request_from_dict -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir logger = logging.getLogger(__name__) @@ -26,12 +26,7 @@ class Scheduler(object): def from_crawler(cls, crawler): settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - if hasattr(dupefilter_cls, 'from_crawler'): - dupefilter = dupefilter_cls.from_crawler(crawler) - elif hasattr(dupefilter_cls, 'from_settings'): - dupefilter = dupefilter_cls.from_settings(crawler.settings) - else: - dupefilter = dupefilter_cls() + dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) From d6d3e87e3a4fd306829a84f934a971fd4e337a26 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 27 Jul 2018 14:47:52 -0300 Subject: [PATCH 0108/1853] Rename test file --- ..._spidermiddleware.py => test_spidermiddleware_output_chain.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_spidermiddleware.py => test_spidermiddleware_output_chain.py} (100%) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware_output_chain.py similarity index 100% rename from tests/test_spidermiddleware.py rename to tests/test_spidermiddleware_output_chain.py From 801d3c07b4b7e57d50429e714a8255e7747568f4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 27 Jul 2018 15:06:25 -0300 Subject: [PATCH 0109/1853] Fix bad exception handling, add tests --- scrapy/core/spidermw.py | 2 + tests/test_spidermiddleware_invalid_values.py | 82 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/test_spidermiddleware_invalid_values.py diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 8607ed620..1b67af130 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -66,6 +66,8 @@ class SpiderMiddlewareManager(MiddlewareManager): if result is not None: raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \ .format(fname(method), type(result))) + except _InvalidOutput: + raise except: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware_invalid_values.py new file mode 100644 index 000000000..0d9af8951 --- /dev/null +++ b/tests/test_spidermiddleware_invalid_values.py @@ -0,0 +1,82 @@ +from twisted.trial.unittest import TestCase +from twisted.python.failure import Failure + +from scrapy.spiders import Spider +from scrapy.http import Request, Response +from scrapy.exceptions import _InvalidOutput +from scrapy.utils.test import get_crawler +from scrapy.core.spidermw import SpiderMiddlewareManager +from tests import mock + + +class SpiderMiddlewareTestCase(TestCase): + + def setUp(self): + self.request = Request('http://example.com/index.html') + self.response = Response(self.request.url, request=self.request) + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) + + def _scrape_response(self): + """Execute spider mw manager's scrape_response method and return the result. + Raise exception in case of failure. + """ + scrape_func = mock.MagicMock() + dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider) + # catch deferred result and return the value + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + ret = results[0] + return ret + + +class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): + """Invalid return value for process_spider_input method""" + + def test_invalid_process_spider_input(self): + + class InvalidProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, _InvalidOutput) + + +class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): + """Invalid return value for process_spider_output method""" + + def test_invalid_process_spider_output(self): + + class InvalidProcessSpiderOutputMiddleware: + def process_spider_output(self, response, result, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, _InvalidOutput) + + +class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): + """Invalid return value for process_spider_exception method""" + + def test_invalid_process_spider_exception(self): + + class InvalidProcessSpiderOutputExceptionMiddleware: + def process_spider_exception(self, response, exception, spider): + return 1 + + class RaiseExceptionProcessSpiderOutputMiddleware: + def process_spider_output(self, response, result, spider): + raise Exception() + + self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) + self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, _InvalidOutput) From 980be4cb4b70c71e5502c8f31c85882540f3f078 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 31 Jul 2018 19:05:04 +0800 Subject: [PATCH 0110/1853] provide better error message when disabling s3 exporter --- scrapy/extensions/feedexport.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7c7db387e..22ebf3b3f 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -269,9 +269,10 @@ class FeedExporter(object): try: self._get_storage(uri) return True - except NotConfigured: - logger.error("Disabled feed storage scheme: %(scheme)s", - {'scheme': scheme}) + except NotConfigured as e: + logger.error("Disabled feed storage scheme: %(scheme)s. " + "Reason: %(reason)s", + {'scheme': scheme, 'reason': str(e)}) else: logger.error("Unknown feed storage scheme: %(scheme)s", {'scheme': scheme}) From c87a4f5c6fd97d22345bb444bc90b56ef0624aa5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 1 Aug 2018 01:45:16 +0500 Subject: [PATCH 0111/1853] remove unused imports from scrapy/settings/__init__.py This is a follow-up to https://github.com/scrapy/scrapy/pull/3327 --- scrapy/settings/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 7d6d20164..14c93bef2 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,13 +1,10 @@ import six import json import copy -import warnings from collections import MutableMapping from importlib import import_module from pprint import pformat -from scrapy.exceptions import ScrapyDeprecationWarning - from . import default_settings From 8c55f5eb159ae85d468a89b74ffaff3e824144ab Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Aug 2018 15:16:26 -0300 Subject: [PATCH 0112/1853] Simplify check for re-raised exception. Add tests. --- scrapy/core/spidermw.py | 4 ++-- ...lid_values.py => test_spidermiddleware.py} | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) rename tests/{test_spidermiddleware_invalid_values.py => test_spidermiddleware.py} (79%) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1b67af130..4268c91d6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): yield r except Exception as ex: exception_result = process_spider_exception(Failure(ex), index+1) - if exception_result is None or isinstance(exception_result, Failure): + if isinstance(exception_result, Failure): raise recovered.extend(exception_result) @@ -115,7 +115,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = method(response=response, result=result, spider=spider) except Exception as ex: exception_result = process_spider_exception(Failure(ex), method_index+1) - if exception_result is None or isinstance(exception_result, Failure): + if isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware.py similarity index 79% rename from tests/test_spidermiddleware_invalid_values.py rename to tests/test_spidermiddleware.py index 0d9af8951..54756f2ff 100644 --- a/tests/test_spidermiddleware_invalid_values.py +++ b/tests/test_spidermiddleware.py @@ -80,3 +80,23 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): result = self._scrape_response() self.assertIsInstance(result, Failure) self.assertIsInstance(result.value, _InvalidOutput) + + +class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): + """Re raise the exception by returning None""" + + def test_process_spider_exception_return_none(self): + + class ProcessSpiderOutputExceptionReturnNoneMiddleware: + def process_spider_exception(self, response, exception, spider): + return None + + class RaiseExceptionProcessSpiderOutputMiddleware: + def process_spider_output(self, response, result, spider): + 1/0 + + self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware()) + self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, ZeroDivisionError) From 40449fa0eb707bac1ae2b78f0f812372e90f17b7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Aug 2018 18:20:25 -0300 Subject: [PATCH 0113/1853] Update docs, add tests, remove FIXME comment --- docs/topics/spider-middleware.rst | 3 +- scrapy/core/scraper.py | 1 - tests/test_spidermiddleware.py | 4 +- tests/test_spidermiddleware_output_chain.py | 43 ++++++++++++++------- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 915833c54..7db623cf4 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -78,7 +78,8 @@ following methods: If it raises an exception, Scrapy won't bother calling any other spider middleware :meth:`process_spider_input` and will call the request - errback. The output of the errback is chained back in the other + errback if there is one, otherwise it will start the :meth:`process_spider_exception` + chain. The output of the errback is chained back in the other direction for :meth:`process_spider_output` to process it, or :meth:`process_spider_exception` if it raised an exception. diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ee1e95a0c..d7fe721fb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -135,7 +135,6 @@ class Scraper(object): return self.spidermw.scrape_response( self.call_spider, request_result, request, spider) else: - # FIXME: don't ignore errors in spider middleware dfd = self.call_spider(request_result, request, spider) return dfd.addErrback( self._log_download_errors, request_result, request, spider) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 54756f2ff..832fd3330 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -87,7 +87,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): def test_process_spider_exception_return_none(self): - class ProcessSpiderOutputExceptionReturnNoneMiddleware: + class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -95,7 +95,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): def process_spider_output(self, response, result, spider): 1/0 - self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware()) + self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() self.assertIsInstance(result, Failure) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 0f5646a72..6f8727a15 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -45,8 +45,13 @@ class RecoveryMiddleware: # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method -class ProcessSpiderInputSpider(Spider): - name = 'ProcessSpiderInputSpider' +class FailProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + spider.logger.info('Middleware: will raise IndexError') + raise IndexError() + +class ProcessSpiderInputSpiderWithoutErrback(Spider): + name = 'ProcessSpiderInputSpiderWithoutErrback' custom_settings = { 'SPIDER_MIDDLEWARES': { # spider @@ -58,23 +63,23 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request(url=self.mockserver.url('/status?n=200'), - callback=self.parse, errback=self.errback) + yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse) def parse(self, response): return {'from': 'callback'} + +class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): + name = 'ProcessSpiderInputSpiderWithErrback' + + def start_requests(self): + yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback) + def errback(self, failure): self.logger.info('Got a Failure on the Request errback') return {'from': 'errback'} -class FailProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): - spider.logger.info('Middleware: will raise IndexError') - raise IndexError() - - # ================================================================================ # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): @@ -278,12 +283,22 @@ class TestSpiderMiddleware(TestCase): self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks - def test_process_spider_input_errback(self): + def test_process_spider_input_without_errback(self): """ - (1) An exception from the process_spider_input chain should not be caught by the - process_spider_exception chain, it should go directly to the Request errback + (1.1) An exception from the process_spider_input chain should be caught by the + process_spider_exception chain from the start if the Request has no errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpider) + log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) + self.assertIn("Middleware: will raise IndexError", str(log1)) + self.assertIn("Middleware: IndexError exception caught", str(log1)) + + @defer.inlineCallbacks + def test_process_spider_input_with_errback(self): + """ + (1.2) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain if the Request has an errback + """ + log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) self.assertNotIn("Middleware: IndexError exception caught", str(log1)) self.assertIn("Middleware: will raise IndexError", str(log1)) self.assertIn("Got a Failure on the Request errback", str(log1)) From d3aa1e86664b02a20522abbb265dc9b91b95f111 Mon Sep 17 00:00:00 2001 From: Raphael Wuillemier Date: Mon, 6 Aug 2018 17:40:31 +0200 Subject: [PATCH 0115/1853] Updated tutorial.rst to include more and up-to-date beginner resources --- docs/intro/tutorial.rst | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 20538e90f..0db6a6218 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -26,15 +26,26 @@ If you're already familiar with other languages, and want to learn Python quickly, we recommend reading through `Dive Into Python 3`_. Alternatively, you can follow the `Python Tutorial`_. -If you're new to programming and want to start with Python, you may find useful -the online book `Learn Python The Hard Way`_. You can also take a look at `this -list of Python resources for non-programmers`_. +If you're new to programming and want to start with Python, the following books +may be useful to you: + +* `Automate the Boring Stuff With Python`_ + +* `How To Think Like a Computer Scientist`_ + +* `Learn Python 3 The Hard Way`_ + +You can also take a look at `this list of Python resources for non-programmers`_, +as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _Dive Into Python 3: http://www.diveintopython3.net .. _Python Tutorial: https://docs.python.org/3/tutorial -.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/ +.. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/ +.. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/ +.. _Learn Python 3 The Hard Way: https://learnpythonthehardway.org/python3/ +.. _suggested resources in the learnpython-subreddit: https://www.reddit.com/r/learnpython/wiki/index#wiki_new_to_python.3F Creating a project From 16dad81715d3970149c0cf7a318e73a0d84be1ff Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Thu, 9 Aug 2018 21:07:25 +0300 Subject: [PATCH 0116/1853] Fix contract errback --- scrapy/contracts/__init__.py | 2 +- tests/test_contracts.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 5eaee3d11..8315d21d2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -84,7 +84,7 @@ class ContractsManager(object): def eb_wrapper(failure): case = _create_testcase(method, 'errback') - exc_info = failure.value, failure.type, failure.getTracebackObject() + exc_info = failure.type, failure.value, failure.getTracebackObject() results.addError(case, exc_info) request.callback = cb_wrapper diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 1cea2afb7..b07cbee1e 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,7 +1,9 @@ from unittest import TextTestResult +from twisted.python import failure from twisted.trial import unittest +from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field @@ -185,3 +187,18 @@ class ContractsManagerTest(unittest.TestCase): self.results) request.callback(response) self.should_fail() + + def test_errback(self): + spider = TestSpider() + response = ResponseMock() + + try: + raise HttpError(response, 'Ignoring non-200 response') + except HttpError: + failure_mock = failure.Failure() + + request = self.conman.from_method(spider.returns_request, self.results) + request.errback(failure_mock) + + self.assertFalse(self.results.failures) + self.assertTrue(self.results.errors) From fb7d4cbce379c4e3fac2ca89b6e0772d0c690935 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 16:08:26 +0300 Subject: [PATCH 0117/1853] Add error handling in contracts --- scrapy/contracts/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 5eaee3d11..18014e290 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -41,7 +41,11 @@ class ContractsManager(object): requests = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) - requests.append(self.from_method(bound_method, results)) + try: + requests.append(self.from_method(bound_method, results)) + except: + case = _create_testcase(bound_method, 'contract') + results.addError(case, sys.exc_info()) return requests From ebbde57eca310d3c5c0f530a2e668572dde4d952 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 17:50:56 +0300 Subject: [PATCH 0118/1853] Add custom contracts tests --- tests/test_contracts.py | 55 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 1cea2afb7..078ef6e0d 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -5,7 +5,7 @@ from twisted.trial import unittest from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field -from scrapy.contracts import ContractsManager +from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, ReturnsContract, @@ -22,6 +22,21 @@ class ResponseMock(object): url = 'http://scrapy.org' +class CustomSuccessContract(Contract): + name = 'custom_success_contract' + + def adjust_request_args(self, args): + args['url'] = 'http://scrapy.org' + return args + + +class CustomFailContract(Contract): + name = 'custom_fail_contract' + + def adjust_request_args(self, args): + raise TypeError('Error in adjust_request_args') + + class TestSpider(Spider): name = 'demo_spider' @@ -99,8 +114,34 @@ class TestSpider(Spider): pass +class CustomContractSuccessSpider(Spider): + name = 'custom_contract_success_spider' + + def parse(self, response): + """ + @custom_success_contract + """ + pass + + +class CustomContractFailSpider(Spider): + name = 'custom_contract_fail_spider' + + def parse(self, response): + """ + @custom_fail_contract + """ + pass + + class ContractsManagerTest(unittest.TestCase): - contracts = [UrlContract, ReturnsContract, ScrapesContract] + contracts = [ + UrlContract, + ReturnsContract, + ScrapesContract, + CustomSuccessContract, + CustomFailContract + ] def setUp(self): self.conman = ContractsManager(self.contracts) @@ -114,6 +155,9 @@ class ContractsManagerTest(unittest.TestCase): self.assertTrue(self.results.failures) self.assertFalse(self.results.errors) + def should_error(self): + self.assertTrue(self.results.errors) + def test_contracts(self): spider = TestSpider() @@ -185,3 +229,10 @@ class ContractsManagerTest(unittest.TestCase): self.results) request.callback(response) self.should_fail() + + def test_custom_contracts(self): + self.conman.from_spider(CustomContractSuccessSpider(), self.results) + self.should_succeed() + + self.conman.from_spider(CustomContractFailSpider(), self.results) + self.should_error() From 76220e8733b1462ecaa5db1cdd4a58ad4e93ceb3 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 18:49:12 +0300 Subject: [PATCH 0119/1853] Use inspect.getmembers in tested_methods_from_spidercls --- scrapy/contracts/__init__.py | 3 ++- tests/test_contracts.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 8315d21d2..de7ac4503 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,6 +1,7 @@ import sys import re from functools import wraps +from inspect import getmembers from unittest import TestCase from scrapy.http import Request @@ -17,7 +18,7 @@ class ContractsManager(object): def tested_methods_from_spidercls(self, spidercls): methods = [] - for key, value in vars(spidercls).items(): + for key, value in getmembers(spidercls): if (callable(value) and value.__doc__ and re.search(r'^\s*@', value.__doc__, re.MULTILINE)): methods.append(key) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b07cbee1e..322d20c4c 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -101,6 +101,10 @@ class TestSpider(Spider): pass +class InheritsTestSpider(TestSpider): + name = 'inherits_demo_spider' + + class ContractsManagerTest(unittest.TestCase): contracts = [UrlContract, ReturnsContract, ScrapesContract] @@ -202,3 +206,9 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + + def test_inherited_contracts(self): + spider = InheritsTestSpider() + + requests = self.conman.from_spider(spider, self.results) + self.assertTrue(requests) From 8fc017d345af1e6fa53334fc9df92ebd8f9dc32e Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 19:25:33 +0300 Subject: [PATCH 0120/1853] Add dont_filter to ContractsManager requests --- scrapy/contracts/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 8315d21d2..2569ab151 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -50,7 +50,12 @@ class ContractsManager(object): if contracts: # calculate request args args, kwargs = get_spec(Request.__init__) + + # Don't filter requests to allow + # testing different callbacks on the same URL. + kwargs['dont_filter'] = True kwargs['callback'] = method + for contract in contracts: kwargs = contract.adjust_request_args(kwargs) From b4b1e4834376e8565d727a1a5087cae31450f931 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 22:18:43 +0300 Subject: [PATCH 0121/1853] Add ability to use FormRequest in contracts --- docs/topics/contracts.rst | 7 +++++-- scrapy/contracts/__init__.py | 7 ++++++- tests/test_contracts.py | 26 ++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ba1421c42..cac52042a 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -86,8 +86,11 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for :class:`~scrapy.http.Request` object. Must return the same or a - modified version of it. + for request object. If ``formdata`` is in ``args``, then + :class:`~scrapy.http.FormRequest` object is created, + otherwise :class:`~scrapy.http.Request` is used. + + Must return the same or a modified version of it. .. method:: Contract.pre_process(response) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 8315d21d2..ca2a8d384 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -3,6 +3,7 @@ import re from functools import wraps from unittest import TestCase +from scrapy import FormRequest from scrapy.http import Request from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec @@ -57,7 +58,11 @@ class ContractsManager(object): # create and prepare request args.remove('self') if set(args).issubset(set(kwargs)): - request = Request(**kwargs) + if 'formdata' in kwargs: + kwargs['method'] = 'POST' + request = FormRequest(**kwargs) + else: + request = Request(**kwargs) # execute pre and post hooks in order for contract in reversed(contracts): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b07cbee1e..f2085f711 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -3,11 +3,12 @@ from unittest import TextTestResult from twisted.python import failure from twisted.trial import unittest +from scrapy import FormRequest from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field -from scrapy.contracts import ContractsManager +from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, ReturnsContract, @@ -24,6 +25,14 @@ class ResponseMock(object): url = 'http://scrapy.org' +class CustomFormContract(Contract): + name = 'custom_form' + + def adjust_request_args(self, args): + args['formdata'] = {'name': 'scrapy'} + return args + + class TestSpider(Spider): name = 'demo_spider' @@ -100,9 +109,16 @@ class TestSpider(Spider): """ pass + def custom_form(self, response): + """ + @url http://scrapy.org + @custom_form + """ + pass + class ContractsManagerTest(unittest.TestCase): - contracts = [UrlContract, ReturnsContract, ScrapesContract] + contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract] def setUp(self): self.conman = ContractsManager(self.contracts) @@ -202,3 +218,9 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + + def test_form_contract(self): + spider = TestSpider() + request = self.conman.from_method(spider.custom_form, self.results) + self.assertEqual(request.method, 'POST') + self.assertIsInstance(request, FormRequest) From 1d25c98eb337763f613d73fd534d3dc5e64eb66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 16:41:53 -0300 Subject: [PATCH 0122/1853] Add appveyor.yml --- appveyor.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..81432be39 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,14 @@ +platform: x86 +version: '{branch}-{build}' +environment: + matrix: + - PYTHON: "C:\\Python36" + TOX_ENV: py36 + +install: + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "pip install -U tox twine wheel" +build: false +test_script: + - "tox -e %TOX_ENV%" From 4c53957f5bd0f64b00d9488f171f0f8a9620af02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 16:56:05 -0300 Subject: [PATCH 0123/1853] Skip leveldb tests on windows --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 8d9ce5231..1e4a4b641 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -3,7 +3,7 @@ pytest-twisted pytest-cov==2.5.1 testfixtures jmespath -leveldb +leveldb; sys_platform != "win32" botocore # optional for shell wrapper tests bpython From dd75297e3fb37800c0ef763003a54081e9fad4cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 16:58:02 -0300 Subject: [PATCH 0124/1853] Run Appveyor CI for master and release branches only, but also PRs --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 81432be39..4f3c69847 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,6 +5,11 @@ environment: - PYTHON: "C:\\Python36" TOX_ENV: py36 +branches: + only: + - master + - /d+\.\d+\.\d+[\w\-]*$/ + install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" From 19ad94105f70a26ce47683d488b66fb77b79067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 3 Jul 2018 17:15:48 -0300 Subject: [PATCH 0125/1853] pywin32 is required to run tests under windows --- tests/requirements-py3.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 1e4a4b641..7c1aacd81 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -9,3 +9,4 @@ botocore bpython ipython brotlipy +pywin32; sys_platform == "win32" From 152fde70b12f3c6e0e73230370f79f4b4a7ea906 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 2 Feb 2016 18:23:23 +0000 Subject: [PATCH 0126/1853] Fix FTPTestCase by using Windows-friendly temporary file name --- tests/test_downloader_handlers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c91be2c0c..fe76989f4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,7 +1,8 @@ import os import six -import contextlib import shutil +import tempfile +import contextlib try: from unittest import mock except ImportError: @@ -913,7 +914,8 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - local_fname = b"/tmp/file.txt" + f, local_fname = tempfile.mkstemp() + os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, @@ -922,7 +924,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']}) + self.assertEqual(r.headers, {b'Local Filename': [local_fname], + b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") From 57a1d66c61423c85703a4008d351e9d566580810 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 16:56:43 +0000 Subject: [PATCH 0127/1853] Fix test issues caused by Windows pipe buffer filling up --- tests/test_commands.py | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7d9071b64..84c38c0e9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,18 +3,17 @@ import os import sys import subprocess import tempfile -from time import sleep from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp from contextlib import contextmanager +from threading import Timer from twisted.trial import unittest from twisted.internet import defer import scrapy from scrapy.utils.python import to_native_str -from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest @@ -46,16 +45,18 @@ class ProjectTest(unittest.TestCase): stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_kwargs) - waited = 0 - interval = 0.2 - while p.poll() is None: - sleep(interval) - waited += interval - if waited > 15: - p.kill() - assert False, 'Command took too much time to complete' + def kill_proc(): + p.kill() + assert False, 'Command took too much time to complete' - return p + timer = Timer(15, kill_proc) + try: + timer.start() + stdout, stderr = p.communicate() + finally: + timer.cancel() + + return to_native_str(stdout), to_native_str(stderr) class StartprojectTest(ProjectTest): @@ -111,8 +112,7 @@ class StartprojectTemplatesTest(ProjectTest): assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] - p = self.proc('startproject', self.project_name, *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + out, err = self.proc('startproject', self.project_name, *args) self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -140,12 +140,10 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = ['--template=%s' % tplname] if tplname else [] spname = 'test_spider' - p = self.proc('genspider', spname, 'test.com', *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - p = self.proc('genspider', spname, 'test.com', *args) - out = to_native_str(retry_on_eintr(p.stdout.read)) + out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) def test_template_basic(self): @@ -212,8 +210,8 @@ class MySpider(scrapy.Spider): return self.proc('runspider', fname, *args) def get_log(self, code, name='myspider.py', args=()): - p = self.runspider(code, name=name, args=args) - return to_native_str(p.stderr.read()) + stdout, stderr = self.runspider(code, name=name, args=args) + return stderr def test_runspider(self): log = self.get_log(self.debug_log_spider) @@ -279,14 +277,17 @@ class MySpider(scrapy.Spider): self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): - p = self.proc('runspider', 'some_non_existent_file') - log = to_native_str(p.stderr.read()) + _, log = self.proc('runspider', 'some_non_existent_file') self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) + +class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + command = 'parse' + def test_start_requests_errors(self): log = self.get_log(""" import scrapy @@ -304,8 +305,7 @@ class BadSpider(scrapy.Spider): class BenchCommandTest(CommandTest): def test_run(self): - p = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', - '-s', 'CLOSESPIDER_TIMEOUT=0.01') - log = to_native_str(p.stderr.read()) + _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', + '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) From ed8255bde04d7e07747792022858f161c2296096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 6 Jul 2018 17:53:56 -0300 Subject: [PATCH 0128/1853] Fix merge issues with stderr/out fixes for windows buffering --- tests/test_commands.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 84c38c0e9..4963ef99c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -56,7 +56,7 @@ class ProjectTest(unittest.TestCase): finally: timer.cancel() - return to_native_str(stdout), to_native_str(stderr) + return p, to_native_str(stdout), to_native_str(stderr) class StartprojectTest(ProjectTest): @@ -112,7 +112,7 @@ class StartprojectTemplatesTest(ProjectTest): assert exists(join(self.tmpl_proj, 'root_template')) args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] - out, err = self.proc('startproject', self.project_name, *args) + p, out, err = self.proc('startproject', self.project_name, *args) self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -140,10 +140,10 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = ['--template=%s' % tplname] if tplname else [] spname = 'test_spider' - out, err = self.proc('genspider', spname, 'test.com', *args) + p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - out, err = self.proc('genspider', spname, 'test.com', *args) + p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) def test_template_basic(self): @@ -210,7 +210,7 @@ class MySpider(scrapy.Spider): return self.proc('runspider', fname, *args) def get_log(self, code, name='myspider.py', args=()): - stdout, stderr = self.runspider(code, name=name, args=args) + p, stdout, stderr = self.runspider(code, name=name, args=args) return stderr def test_runspider(self): @@ -221,12 +221,12 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Spider closed (finished)", log) def test_run_fail_spider(self): - proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) ret = proc.returncode self.assertNotEqual(ret, 0) def test_run_good_spider(self): - proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) ret = proc.returncode self.assertEqual(ret, 0) @@ -277,7 +277,7 @@ class MySpider(scrapy.Spider): self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): - _, log = self.proc('runspider', 'some_non_existent_file') + _, _, log = self.proc('runspider', 'some_non_existent_file') self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): @@ -305,7 +305,7 @@ class BadSpider(scrapy.Spider): class BenchCommandTest(CommandTest): def test_run(self): - _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', + _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) From 034152961d187930eb659bef9e87ceb3e589c103 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 18:12:08 +0000 Subject: [PATCH 0129/1853] Fix Feedexport test in Windows by using proper file URI --- tests/test_feedexport.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2b57449d9..6eefa14bf 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2,11 +2,12 @@ from __future__ import absolute_import import os import csv import json +import warnings from io import BytesIO import tempfile import shutil -from six.moves.urllib.parse import urlparse -import warnings +from six.moves.urllib.parse import urljoin, urlparse +from six.moves.urllib.request import pathname2url from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -226,9 +227,10 @@ class FeedExportTest(unittest.TestCase): def run_and_export(self, spider_cls, settings=None): """ Run spider with specified settings; return exported data. """ tmpdir = tempfile.mkdtemp() - res_name = tmpdir + '/res' + res_path = os.path.join(tmpdir, 'res') + res_uri = urljoin('file:', pathname2url(res_path)) defaults = { - 'FEED_URI': 'file://' + res_name, + 'FEED_URI': res_uri, 'FEED_FORMAT': 'csv', } defaults.update(settings or {}) @@ -238,7 +240,7 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(res_name, 'rb') as f: + with open(res_path, 'rb') as f: defer.returnValue(f.read()) finally: From 22505a34a9d0095dfc6e133bbb79d3b7aa651082 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 18:42:51 +0000 Subject: [PATCH 0130/1853] Fix cmdline profiling test on Windows by using proper path composing --- tests/test_cmdline/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 10076bbca..68dfb1cca 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -52,7 +52,8 @@ class CmdlineTest(unittest.TestCase): stats.print_stats() out.seek(0) stats = out.read() - self.assertIn('scrapy/commands/version.py', stats) + self.assertIn(os.path.join('scrapy', 'commands', 'version.py'), + stats) self.assertIn('tottime', stats) finally: shutil.rmtree(path) From fb09148c91f118e3a71243e527a0d39294ba59cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 6 Jul 2018 18:10:56 -0300 Subject: [PATCH 0131/1853] Fix bad merge on ParseCommandTest --- tests/test_commands.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 4963ef99c..78aa2a776 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -284,10 +284,6 @@ class MySpider(scrapy.Spider): log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) - -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' - def test_start_requests_errors(self): log = self.get_log(""" import scrapy From a21abac743865b628e52f11882db7b7ab70e0342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 10:15:20 -0300 Subject: [PATCH 0132/1853] fix ftp tests on windows --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fe76989f4..2f8973054 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -915,6 +915,7 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_local_filename(self): f, local_fname = tempfile.mkstemp() + local_fname = to_bytes(local_fname) os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) From ed068e59b746de6395c33f6f0f5454e69e88d1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 10:28:07 -0300 Subject: [PATCH 0133/1853] Cache pip cache and do not rebuild tags on appveyor and travis --- appveyor.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 4f3c69847..93cfd469e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,7 +13,12 @@ branches: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" - - "pip install -U tox twine wheel" + - "pip install -U tox" + build: false +skip_tags: true test_script: - "tox -e %TOX_ENV%" + +cache: + - '%LOCALAPPDATA%\pip\cache' From 0e532e3dd8b4dd07b1a287d6321c1e6ae6786c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 10:46:15 -0300 Subject: [PATCH 0134/1853] Creating a connection to 0.0.0.0 fails on windows but not on linux nor mac --- tests/mockserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index f36ce3c44..bf62fe907 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -209,7 +209,7 @@ class MockServer(): time.sleep(0.2) def url(self, path, is_secure=False): - host = self.http_address + host = self.http_address.replace('0.0.0.0', '127.0.0.1') if is_secure: host = self.https_address return host + path From ca53a8699a8cfc2a0a3be4a65d91bada8a43f94c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 7 Jul 2018 11:07:05 -0300 Subject: [PATCH 0135/1853] Fix presentation of template directory in startproject command --- scrapy/commands/startproject.py | 4 ++-- tests/test_commands.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index c17aaf442..67337c26e 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -107,8 +107,8 @@ class Command(ScrapyCommand): string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project %r, using template directory %r, created in:" % \ - (project_name, self.templates_dir)) + print("New Scrapy project '%s', using template directory '%s', " + "created in:" % (project_name, self.templates_dir)) print(" %s\n" % abspath(project_dir)) print("You can start your first spider with:") print(" cd %s" % project_dir) diff --git a/tests/test_commands.py b/tests/test_commands.py index 78aa2a776..b8445ae6c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -113,7 +113,8 @@ class StartprojectTemplatesTest(ProjectTest): args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out) + self.assertIn("New Scrapy project '%s', using template directory" + % self.project_name, out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) From cb281757500b2e114fb554aa3bf618e3a8f8eb71 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 3 Feb 2016 18:35:35 +0000 Subject: [PATCH 0136/1853] Fix csviter tests by explicitly using newline only --- tests/test_utils_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index b2e8610f8..f953076b8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,7 +7,7 @@ from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = u"foo" + os.linesep + u"bar" +FOOBAR_NL = u"foo\nbar" class XmliterTestCase(unittest.TestCase): From d93d960319e22badccd68499df11f2a728dbbc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 15 Aug 2018 01:53:20 -0300 Subject: [PATCH 0137/1853] Fix test_utils_project under Windows --- tests/test_utils_project.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 7e2caace8..bd74b0c34 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -25,8 +25,12 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEqual('.scrapy/somepath', data_path('somepath')) - self.assertEqual('/absolute/path', data_path('/absolute/path')) + self.assertEqual( + os.path.join('.scrapy', 'somepath'), + data_path('somepath') + ) + abspath = os.path.join(os.path.sep, 'absolute', 'path') + self.assertEqual(abspath, data_path(abspath)) def test_data_path_inside_project(self): with inside_a_project() as proj_path: @@ -35,4 +39,5 @@ class ProjectUtilsTest(unittest.TestCase): os.path.realpath(expected), os.path.realpath(data_path('somepath')) ) - self.assertEqual('/absolute/path', data_path('/absolute/path')) + abspath = os.path.join(os.path.sep, 'absolute', 'path') + self.assertEqual(abspath, data_path(abspath)) From 96517cb7de93da990f4f7dcb1cb7b7129e8c2064 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 15 Aug 2018 01:08:40 -0700 Subject: [PATCH 0138/1853] Fix test_command_parse under windows --- tests/test_command_parse.py | 38 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 66dd17110..02037b866 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,3 +1,4 @@ +import os from os.path import join, abspath from twisted.trial import unittest from twisted.internet import defer @@ -7,6 +8,11 @@ from scrapy.utils.python import to_native_str from tests.test_commands import CommandTest +def _textmode(bstr): + """Normalize input the same as writing to a file + and reading from it in text mode""" + return to_native_str(bstr).replace(os.linesep, '\n') + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' @@ -97,7 +103,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '-a', 'test_arg=1', '-c', 'parse', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_with_meta(self): @@ -106,13 +112,13 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--meta', raw_json_string, '-c', 'parse_request_with_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) _, _, stderr = yield self.execute(['--spider', self.spider_name, '-m', raw_json_string, '-c', 'parse_request_with_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks @@ -120,7 +126,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} _, _, stderr = yield self.execute(['--spider', self.spider_name, '-c', 'parse_request_without_meta', self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks @@ -129,29 +135,29 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} '--pipelines', '-c', 'parse', self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) + self.assertIn("INFO: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find callback""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): @@ -159,7 +165,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): @@ -167,7 +173,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) @defer.inlineCallbacks def test_spider_with_no_rules_attribute(self): @@ -175,15 +181,15 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -191,5 +197,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] ) - self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) + self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) From e7fe243c3e6f71660620441ec6a8ca4605d4333b Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 15 Aug 2018 01:09:23 -0700 Subject: [PATCH 0139/1853] Fix test_crawler under windows --- tests/test_crawler.py | 46 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6a8e11363..0aeb12e58 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,10 +1,9 @@ import logging import tempfile import warnings -import unittest from twisted.internet import defer -import twisted.trial.unittest +from twisted.trial import unittest import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess @@ -94,26 +93,29 @@ class CrawlerLoggingTestCase(unittest.TestCase): assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): - with tempfile.NamedTemporaryFile() as log_file: - class MySpider(scrapy.Spider): - name = 'spider' - custom_settings = { - 'LOG_LEVEL': 'INFO', - 'LOG_FILE': log_file.name, - # disable telnet if not available to avoid an extra warning - 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, - } + log_file = self.mktemp() + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_LEVEL': 'INFO', + 'LOG_FILE': log_file, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + crawler = Crawler(MySpider, {}) + self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value('log_count/INFO') + logging.debug('debug message') + logging.info('info message') + logging.warning('warning message') + logging.error('error message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf8') - configure_logging() - self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) - crawler = Crawler(MySpider, {}) - self.assertEqual(get_scrapy_root_handler().level, logging.INFO) - info_count = crawler.stats.get_value('log_count/INFO') - logging.debug('debug message') - logging.info('info message') - logging.warning('warning message') - logging.error('error message') - logged = log_file.read().decode('utf8') self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -203,7 +205,7 @@ class NoRequestsSpider(scrapy.Spider): return [] -class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): +class CrawlerRunnerHasSpider(unittest.TestCase): @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): From a304d6b692664f06c67fb23eb08fd985eda72e21 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 15 Aug 2018 02:02:20 -0700 Subject: [PATCH 0140/1853] Workaround to pass tests/test_feedexporter.py under windows --- tests/test_feedexport.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6eefa14bf..76452d450 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -241,10 +241,18 @@ class FeedExportTest(unittest.TestCase): yield runner.crawl(spider_cls) with open(res_path, 'rb') as f: - defer.returnValue(f.read()) + content = f.read() finally: - shutil.rmtree(tmpdir) + # FIXME: Windows fails to remove the file because FeedExporter + # keeps a reference to the temporal file even after + # the spider finished. + try: + shutil.rmtree(tmpdir) + except OSError: + pass + + defer.returnValue(content) @defer.inlineCallbacks def exported_data(self, items, settings): From 4eaf8690b14527d8d74ddd8103810de7c4dcdee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 15 Aug 2018 08:54:18 -0300 Subject: [PATCH 0141/1853] Twisted's unittest.Testcase assertRaiess can't be used as context manager --- tests/test_crawler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0aeb12e58..268948a70 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -143,9 +143,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): settings = Settings({ 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' }) - with warnings.catch_warnings(record=True) as w, \ - self.assertRaises(AttributeError): - CrawlerRunner(settings) + with warnings.catch_warnings(record=True) as w: + self.assertRaises(AttributeError, CrawlerRunner, settings) self.assertEqual(len(w), 1) self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message)) self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) From 38608bc2495189bd7de2dc48c0036df217138377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 15 Aug 2018 11:59:09 -0300 Subject: [PATCH 0142/1853] Use ignore_errors option from rmtree --- tests/test_feedexport.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 76452d450..e46c8c14e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -244,13 +244,7 @@ class FeedExportTest(unittest.TestCase): content = f.read() finally: - # FIXME: Windows fails to remove the file because FeedExporter - # keeps a reference to the temporal file even after - # the spider finished. - try: - shutil.rmtree(tmpdir) - except OSError: - pass + shutil.rmtree(tmpdir, ignore_errors=True) defer.returnValue(content) From 4de493efdd80d8ff85a78009dcc120ac46b9c55c Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:24:00 +0300 Subject: [PATCH 0143/1853] Add test_same_url --- tests/test_contracts.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b07cbee1e..e4f1a777b 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,8 +1,10 @@ from unittest import TextTestResult +from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest +from scrapy.crawler import CrawlerRunner from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request @@ -101,6 +103,29 @@ class TestSpider(Spider): pass +class TestSameUrlSpider(Spider): + + name = 'test_same_url' + + def __init__(self, *args, **kwargs): + super(TestSameUrlSpider, self).__init__(*args, **kwargs) + self.visited = 0 + + def parse_first(self, response): + """first callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + def parse_second(self, response): + """second callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + class ContractsManagerTest(unittest.TestCase): contracts = [UrlContract, ReturnsContract, ScrapesContract] @@ -177,14 +202,12 @@ class ContractsManagerTest(unittest.TestCase): self.should_succeed() # scrapes_item_fail - request = self.conman.from_method(spider.scrapes_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_item_fail, self.results) request.callback(response) self.should_fail() # scrapes_dict_item_fail - request = self.conman.from_method(spider.scrapes_dict_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results) request.callback(response) self.should_fail() @@ -202,3 +225,12 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + + @defer.inlineCallbacks + def test_same_url(self): + TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) + + crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + yield crawler.crawl() + + self.assertEqual(crawler.spider.visited, 2) From 2cb4decb6ad52548b3441c877235c540cef12082 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:36:10 +0300 Subject: [PATCH 0144/1853] Move TestSameUrlSpider to test method --- tests/test_contracts.py | 45 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index e4f1a777b..430d89253 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -103,29 +103,6 @@ class TestSpider(Spider): pass -class TestSameUrlSpider(Spider): - - name = 'test_same_url' - - def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) - self.visited = 0 - - def parse_first(self, response): - """first callback - @url http://scrapy.org - """ - self.visited += 1 - return TestItem() - - def parse_second(self, response): - """second callback - @url http://scrapy.org - """ - self.visited += 1 - return TestItem() - - class ContractsManagerTest(unittest.TestCase): contracts = [UrlContract, ReturnsContract, ScrapesContract] @@ -228,6 +205,28 @@ class ContractsManagerTest(unittest.TestCase): @defer.inlineCallbacks def test_same_url(self): + + class TestSameUrlSpider(Spider): + name = 'test_same_url' + + def __init__(self, *args, **kwargs): + super(TestSameUrlSpider, self).__init__(*args, **kwargs) + self.visited = 0 + + def parse_first(self, response): + """first callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + def parse_second(self, response): + """second callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) From ddd69f4c10578975658ef6ac450cbe1bed85df80 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:39:43 +0300 Subject: [PATCH 0145/1853] Use MockServer in test_same_url --- tests/test_contracts.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 430d89253..223a926f1 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -15,6 +15,7 @@ from scrapy.contracts.default import ( ReturnsContract, ScrapesContract, ) +from tests.mockserver import MockServer class TestItem(Item): @@ -213,6 +214,9 @@ class ContractsManagerTest(unittest.TestCase): super(TestSameUrlSpider, self).__init__(*args, **kwargs) self.visited = 0 + def start_requests(s): + return self.conman.from_spider(s, self.results) + def parse_first(self, response): """first callback @url http://scrapy.org @@ -227,9 +231,8 @@ class ContractsManagerTest(unittest.TestCase): self.visited += 1 return TestItem() - TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) - crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) - yield crawler.crawl() + with MockServer() as mockserver: + yield crawler.crawl(mockserver=mockserver) self.assertEqual(crawler.spider.visited, 2) From 2b212d426668c02704bc23c4e988fa108e0738ef Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:06 +0000 Subject: [PATCH 0146/1853] ignore cache for pytests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 406146e5f..ff6e2ea65 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ dist .idea htmlcov/ .coverage +.pytest_cache/ .coverage.* .cache/ From d95762db7c9abe59d8edfc8c17397a5a5bbfc661 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:24 +0000 Subject: [PATCH 0147/1853] new signal --- scrapy/signals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/signals.py b/scrapy/signals.py index e36c27203..c0e4bb74e 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -13,6 +13,7 @@ spider_closed = object() spider_error = object() request_scheduled = object() request_dropped = object() +request_reached_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() From 597b8a97ad468123432dba3d6f2c3dd943f4aa36 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:42 +0000 Subject: [PATCH 0148/1853] documentation for new signal --- docs/topics/signals.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d40c0e1df..cf7b8db2f 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -279,6 +279,22 @@ request_dropped :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_reached_downloader +--------------------------- + +.. signal:: request_reached_downloader +.. function:: request_reached_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request`, reached downloader. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- From afb1458bd3b75fad78f053ac062910052971cdfc Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:54 +0000 Subject: [PATCH 0149/1853] tests for new signal --- tests/test_engine.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_engine.py b/tests/test_engine.py index 719c0c60c..856465161 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -103,6 +103,7 @@ class CrawlerRun(object): self.respplug = [] self.reqplug = [] self.reqdropped = [] + self.reqreached = [] self.itemerror = [] self.itemresp = [] self.signals_catched = {} @@ -124,6 +125,7 @@ class CrawlerRun(object): self.crawler.signals.connect(self.item_error, signals.item_error) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) + self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) self.crawler.crawl(start_urls=start_urls) self.spider = self.crawler.spider @@ -155,6 +157,9 @@ class CrawlerRun(object): def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) + def request_reached(self, request, spider): + self.reqreached.append((request, spider)) + def request_dropped(self, request, spider): self.reqdropped.append((request, spider)) @@ -212,6 +217,8 @@ class EngineTest(unittest.TestCase): responses_count = len(self.run.respplug) self.assertEqual(scheduled_requests_count, dropped_requests_count + responses_count) + self.assertEqual(len(self.run.reqreached), + responses_count) def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) @@ -219,6 +226,7 @@ class EngineTest(unittest.TestCase): def _assert_downloaded_responses(self): # response tests self.assertEqual(8, len(self.run.respplug)) + self.assertEqual(8, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': From 561ad3b63c539b560bbaed1df37d0b60043ab3b3 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:40:24 +0000 Subject: [PATCH 0150/1853] emit new signal --- scrapy/core/downloader/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d835e65f7..59c3ad074 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -129,6 +129,9 @@ class Downloader(object): return response slot.active.add(request) + self.signals.send_catch_log(signal=signals.request_reached_downloader, + request=request, + spider=spider) deferred = defer.Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) From 5bac43676425d25169830d4410db60a98a11911f Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Fri, 17 Aug 2018 15:07:37 -0300 Subject: [PATCH 0151/1853] Make lazy loading Download Handlers optional --- scrapy/core/downloader/handlers/__init__.py | 14 ++++++++++-- scrapy/core/downloader/handlers/s3.py | 1 + tests/test_downloader_handlers.py | 25 ++++++++++++++++----- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index bc5cd742e..ebe6f5b78 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -24,6 +24,13 @@ class DownloadHandlers(object): crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath + for scheme in self._schemes: + path = self._schemes[scheme] + dhcls = load_object(path) + lazy = getattr(dhcls, 'lazy', False) + if lazy: + continue + self._load_handler(scheme, dhcls) crawler.signals.connect(self._close, signals.engine_stopped) @@ -40,8 +47,12 @@ class DownloadHandlers(object): return None path = self._schemes[scheme] + dhcls = load_object(path) + self._load_handler(scheme, dhcls) + return self._handlers[scheme] + + def _load_handler(self, scheme, dhcls): try: - dhcls = load_object(path) dh = dhcls(self._crawler.settings) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) @@ -54,7 +65,6 @@ class DownloadHandlers(object): return None else: self._handlers[scheme] = dh - return self._handlers[scheme] def download_request(self, request, spider): scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d8bbdd326..e723e616d 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -31,6 +31,7 @@ def _get_boto_connection(): class S3DownloadHandler(object): + lazy = True def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HTTPDownloadHandler, **kw): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2f8973054..116942ebe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -41,12 +41,20 @@ from scrapy.exceptions import NotConfigured from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider + class DummyDH(object): def __init__(self, crawler): pass +class DummyLazyDH(object): + lazy = True + + def __init__(self, crawler): + pass + + class OffDH(object): def __init__(self, crawler): @@ -60,8 +68,6 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) self.assertIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) @@ -70,8 +76,6 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) @@ -80,11 +84,22 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers + for scheme in handlers: # force load handlers dh._get_handler(scheme) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) + def test_lazy_handlers(self): + handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) + self.assertIn('scheme', dh._schemes) + self.assertNotIn('scheme', dh._handlers) + for scheme in handlers: # force load lazy handler + dh._get_handler(scheme) + self.assertIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) + class FileTestCase(unittest.TestCase): From e2de0a7203b99d8fc71e62539fdd75dad439982a Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 18 Aug 2018 15:24:30 +0300 Subject: [PATCH 0152/1853] Use except Exception --- scrapy/contracts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index c62df5ab0..18c59ff22 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -44,7 +44,7 @@ class ContractsManager(object): bound_method = spider.__getattribute__(method) try: requests.append(self.from_method(bound_method, results)) - except: + except Exception: case = _create_testcase(bound_method, 'contract') results.addError(case, sys.exc_info()) From 0467737cf0bc4c5603bd63a87e220482eb724600 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 18 Aug 2018 15:42:21 +0300 Subject: [PATCH 0153/1853] Fix mockserver usage --- tests/test_contracts.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c5820f256..0a216b745 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -222,22 +222,20 @@ class ContractsManagerTest(unittest.TestCase): return self.conman.from_spider(s, self.results) def parse_first(self, response): - """first callback - @url {} - """.format(self.mockserver.url('/status?n=200')) self.visited += 1 return TestItem() def parse_second(self, response): - """second callback - @url {} - """.format(self.mockserver.url('/status?n=200')) self.visited += 1 return TestItem() - crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) with MockServer() as mockserver: - yield crawler.crawl(mockserver=mockserver) + mock_endpoint = mockserver.url('/status?n=200') + TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint) + TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint) + + crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + yield crawler.crawl() self.assertEqual(crawler.spider.visited, 2) From 57824600a8295f0d8537de2a8d1c3eb9977ed36d Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sun, 19 Aug 2018 16:56:41 +0300 Subject: [PATCH 0154/1853] Use six.get_unbound_function in test_same_url --- tests/test_contracts.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 0a216b745..b4209e1f6 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,5 +1,6 @@ from unittest import TextTestResult +from six import get_unbound_function from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -230,9 +231,10 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - mock_endpoint = mockserver.url('/status?n=200') - TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint) - TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint) + contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + + get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc + get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) yield crawler.crawl() From 167211ffb0e7fa7756483aa9f20e1ab5589c7c4a Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 20 Aug 2018 15:54:04 -0300 Subject: [PATCH 0155/1853] Default is lazy, load_object exception handling, code improvements --- scrapy/core/downloader/handlers/__init__.py | 22 +++++++++------------ scrapy/core/downloader/handlers/datauri.py | 2 ++ scrapy/core/downloader/handlers/file.py | 2 ++ scrapy/core/downloader/handlers/ftp.py | 3 +++ scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/handlers/http11.py | 1 + scrapy/core/downloader/handlers/s3.py | 1 - tests/test_downloader_handlers.py | 4 +++- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ebe6f5b78..0b55d32fa 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -24,13 +24,7 @@ class DownloadHandlers(object): crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath - for scheme in self._schemes: - path = self._schemes[scheme] - dhcls = load_object(path) - lazy = getattr(dhcls, 'lazy', False) - if lazy: - continue - self._load_handler(scheme, dhcls) + self._load_handler(scheme, skip_lazy=True) crawler.signals.connect(self._close, signals.engine_stopped) @@ -46,13 +40,14 @@ class DownloadHandlers(object): self._notconfigured[scheme] = 'no handler available for that scheme' return None - path = self._schemes[scheme] - dhcls = load_object(path) - self._load_handler(scheme, dhcls) - return self._handlers[scheme] + return self._load_handler(scheme) - def _load_handler(self, scheme, dhcls): + def _load_handler(self, scheme, skip_lazy=False): + path = self._schemes[scheme] try: + dhcls = load_object(path) + if skip_lazy and getattr(dhcls, 'lazy', True): + return None dh = dhcls(self._crawler.settings) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) @@ -60,11 +55,12 @@ class DownloadHandlers(object): except Exception as ex: logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"', {"clspath": path, "scheme": scheme}, - exc_info=True, extra={'crawler': self._crawler}) + exc_info=True, extra={'crawler': self._crawler}) self._notconfigured[scheme] = str(ex) return None else: self._handlers[scheme] = dh + return dh def download_request(self, request, spider): scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index d102f2b73..ad25beb3b 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers class DataURIDownloadHandler(object): + lazy = False + def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 9346ce08d..23f25d28d 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers + class FileDownloadHandler(object): + lazy = False def __init__(self, settings): pass diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 933bc7e8d..c342d4ab1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol): self.body.close() if self.filename else self.body.seek(0) _CODE_RE = re.compile("\d+") + + class FTPDownloadHandler(object): + lazy = False CODE_MAPPING = { "550": 404, diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 0322bbe49..d875fb1e4 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode class HTTP10DownloadHandler(object): + lazy = False def __init__(self, settings): self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 038db7b47..0673188a1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler(object): + lazy = False def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index e723e616d..d8bbdd326 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -31,7 +31,6 @@ def _get_boto_connection(): class S3DownloadHandler(object): - lazy = True def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HTTPDownloadHandler, **kw): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 116942ebe..0d0829793 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -43,19 +43,21 @@ from tests.spiders import SingleRequestSpider class DummyDH(object): + lazy = False def __init__(self, crawler): pass class DummyLazyDH(object): - lazy = True + # Default is lazy for backwards compatibility def __init__(self, crawler): pass class OffDH(object): + lazy = False def __init__(self, crawler): raise NotConfigured From af555cab23958501f07f8b2771c082ac3f3a5961 Mon Sep 17 00:00:00 2001 From: Raphael Wuillemier Date: Wed, 22 Aug 2018 14:15:53 +0200 Subject: [PATCH 0156/1853] Added general guide for developer tools instead of Firefox and Firebug-sections --- docs/index.rst | 10 +- docs/intro/tutorial.rst | 3 +- docs/topics/_images/inspector_01.png | Bin 0 -> 53922 bytes docs/topics/_images/network_01.png | Bin 0 -> 10720 bytes docs/topics/_images/network_02.png | Bin 0 -> 82702 bytes docs/topics/_images/network_03.png | Bin 0 -> 45506 bytes docs/topics/developer-tools.rst | 248 +++++++++++++++++++++++++++ docs/topics/firebug.rst | 167 ------------------ docs/topics/firefox.rst | 82 --------- 9 files changed, 252 insertions(+), 258 deletions(-) create mode 100644 docs/topics/_images/inspector_01.png create mode 100644 docs/topics/_images/network_01.png create mode 100644 docs/topics/_images/network_02.png create mode 100644 docs/topics/_images/network_03.png create mode 100644 docs/topics/developer-tools.rst delete mode 100644 docs/topics/firebug.rst delete mode 100644 docs/topics/firefox.rst diff --git a/docs/index.rst b/docs/index.rst index 7e8c979c4..0a96aa88e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -151,8 +151,7 @@ Solving specific problems topics/contracts topics/practices topics/broad-crawls - topics/firefox - topics/firebug + topics/developer-tools topics/leaks topics/media-pipeline topics/deploy @@ -175,11 +174,8 @@ Solving specific problems :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. -:doc:`topics/firefox` - Learn how to scrape with Firefox and some useful add-ons. - -:doc:`topics/firebug` - Learn how to scrape efficiently using Firebug. +:doc:`topics/developer-tools` + Learn how to scrape with your browser's developer tools. :doc:`topics/leaks` Learn how to find and get rid of memory leaks in your crawler. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 0db6a6218..fa6dc274d 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -298,8 +298,7 @@ expressions`:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools or extensions like Firebug (see -sections about :ref:`topics-firebug` and :ref:`topics-firefox`). +You can use your browser developer tools (see section about :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. diff --git a/docs/topics/_images/inspector_01.png b/docs/topics/_images/inspector_01.png new file mode 100644 index 0000000000000000000000000000000000000000..edb8795dcb8890bac8311db1445435f969761aa0 GIT binary patch literal 53922 zcmcG#1$0zfmo-{3;!23S6L)tZfe?cbcXxMpcZd)dNJ2az?jcUx-QC^oKNar1U-$pL z?jHT#cr{>9>^gPMmL+q}wJKCzRuUNj4*>)MA-|Op{{R9(!Gk~$6L2qpk^1YJ6W||s z8!1(L5D29M{1+mg4h0_=gmri;BLTY#hlPNEHF6oA`#dC};vnMaU}9|t`ccv^1q>lM z07K%2_WE|FHV&rNR-l*gq~yRj0(e}^+Q!w+)Y!xURD{~k3yh*XkJ=gPf-fa;Fts!U zaa^GS^8vwwKlx~F-UK_1Q|FboGCR@Sw%GPN=W zwc>9E17om%zVB$KZwR`0^56UWGBImQOG7IM&}Dx@3o!Nqe3_!DgM}d||8SHV1R?>w z6&F!-N!?$tRKZkE=X$g@m#@$pjNWMN6zW zt8I^P>q$DceG2=pAP#(tSnMtewqemYN&~#qR}oZr!;z{Rq$d3&=xFt>wIZGQ6hEHA z56uday`Ym}AG}HmnYMX9<;kWU76`WRZaL6Fi=YXMi#vaNy2*HI%XmDeZoFD_KR!K8 z-d;sRAHig?%ti!0FCm44Ty)(cY&`yjh=djr((@V#BP8Td-j5_RD+_{7sVHeX8+>^? ziEqd@%Eel=Pu+V z$(+mKVadhQC@gh7@Evh+b_v^B@HOb@y%Fwmkk9XAtkz~f+id)fKba?d9*6`5<3GQN zMe>Ix1Ha;QV}Qv5FV>H&QruI#A@;@hVRB^Iy(Oiks60IGXoQ5w*4EZ8?(XEgym)-h zkLXKYcMPD+=4PUci;I^UAAnmV$BBsE%%so=x!Y%l+jgt`G>Y75pBs^Ma5xvoPBBTk zy>%0plq4r3(^F9s7l+Qy$(g|+l$4N=xSkpE3bJl#inzb`I6ptn%*%tlxw%1uhDhPG z6qOw$;=dUG_)=3buI%D#tGSR275LFkhxg>1mSJL2QXxB@^`;+F_uoKA_Lr3kYyLDs z#>2-?`)#l@`kmYBzINfFM5E5n^=y+T{;K6sTVQvVV2?wEXgz4G!J|00d)*;xlengQ zmzvFnzP%zCzIolj?nmW6@Ob}qAr|$sH@^IK`&GY`{YceuOU_~lTZdIKQ z#Y*8qk0;UG?w3Uijk0OHfz#6}V`s6c}<08 zlW1>oS67~2?d=RDXDh+PWDE?*#NPMi6OQ7W7i@8BSlVNuwOBHg)g?!R9$LUD>2D3D zx$JyT4EpYQ+y*NpE1O<h=pS{yup{#emdQ z0*iSZ#P6(1P76Y|tI(CR2qx^EE*tU61h=V#2OLkb4$(J$U0q%1o;daIH0MaH4l<60 z>)C3m=H3e>toVPOo3&k@@g@q-j{Mzkd0;Yo5R8_g%hVQtvo}>%T#Ve8$OIgB;7)rj z@E^dFDyEO?KCiE@FE?E+{+!$XQCcdgp+NvF@}LB<2VVl%YZFt`^6M3ME^|xEDm)U9 z&xao!V{$^4Tp88XoE=?V$t8iKS0!{p&}qF~%*^sm#py`M&bkkGL%9>gBU5Kc#n2M%IFoSc(p#}XyBC#w>bRaMCD+Emoka`K^F89zmQ$QiX7 zxwD7HdL7bmih&*RI3JO8S@wku`d+Nngjc9s`sUql&W?_bX=Dtl-DIH|nckH*?3W$7 zpeVupnI`wEGSs!!*4BpA_PVO7iL5trE-u`sXJ_cx*ul-sf_ay=hx0s*x4VVp46B5D zE5Y7TFR5i!WWc*k_x=2WOcXia=6^U=S)s`;cems`(h)GIH z3Y=!wXF0(gd1lLAz{a9RM?V0o;AuLzzlmCKsGl33nj)v5fd290htrgdjEn@}B4T1! zSd1DWAM<2;6ViAc5%>TbwpE39xtuXRxtz0>b6}wwAN?*}?(Lm5;D~^5Nlru4W2M!& zsqUl<3lE&{T5--4i@kcoj<>>l*+kQu*she^i)i-6%33e`sjJ+LO#!@3*U+7C=bht* zY{Brk@H@<_JK$Y|0Oe;1b#?rA($bdcI+m7}6}Ba#sOUf>@c_&+w-QzGy7fy!TpX(f zzPjto?3O{MB5iD3oS#ezCr_28jt&V121e1OucEtqLudUt;8+|L--f>5gq8;WG##}A z?0l_iO3U7~>FVNQWxaRE@{)pHvmTRL&dZ96M!n9C5_qoR1q{+fErWJ5L21(^9^ldj zM>W7cs5Y8&CtszzdZr{wzE4c(8xSWm;5uLcD z_uS>UE%5mAvP=8%l&Iq8&x6g4Z4P3a62T%(#5boryy#SxF*YTRXbn$Bu1FaS4Np!J zKdc3Lmu>_ACys7;7{qT88b<|jkGj0Ip#XJWZR&8fyTn4>9U&TZs;1?o|o)&w<5&p{Ev&80)qWfmUKCx2TWF0R*RCgjg7#F2qgZqhzC+h;1S$*>mmDM!t_sd_FKoJ(o8Ee z#u@P!8mZj2kVArZ`(=#;o=1SOcud3k7vCl%d{SdPJD^X)F7Xfml8PaOu&pi2q6;}O zD*rL4v9YntdPzOMpa9SI5gc6rV|~By6xDe2&5>eo%eFeI!~REh_Ahp)nR{Ls5L456 zoM}dB^KHR`Ilc}Y+GMG=^~o(K(NwX}!;YNG?M~jv?5vsd!sMh9|HJv%HQ?>*@{53{ zet{7H1tBlktm7G#E)seQ5DHjpZtTzeRC3=T{K`b>288({tucm)yc zFZ*3%KR)K65fE7X?wc`x`|ceAEiJ7EFU94}Rz`OMgSyTAwQWbYP!NzsM$SW`y^(-7 zEFV68Ecd)~3O==8Pe$cdCY~&TA%7qsw3dZ|2eLd3BO{R$HY1PgToTqh3OIyB>!12f z)Sx6t-_4Svlb1{GWlz^WMR~#$gT``KiR~5p#o7GPDfGOBCkCin7x%T8< zO)kX{tp>Y~WI(-=bbevATAx8kXep~f>@wtYJOjEM#)%j+Gcy{nc5(MRdAU%R#J3uG zK+XVrMqz0)@BUycU>AF#k!67OR903-Y@&6ilE7Umv$AQ5wgtXYe)7;cU2A?BgY$wW_sWeu*VoG$XN^l~`UfPH1Y4w zudddjz2`$uPopKj1Uu05t7uVlgeL1vX#k0QY%QF6t>zVAim-`H+Ff}WUc1MfdO6vd znXL{Aw(HYCCN;6=9#l!n=k>Z;xH|rvaSsi|(~1d4q@|e*fA9M>Vw5d>9Gs9}zeE8C z{Iz;lS5`KRSzDsrvS=pw`J`!!Ryu)R1XvJor;tEIY_iv{;alz&5U-=3T%dj$4|SDj zHe`OFT$!_O>HNxOjE0LF3S^F%rf@U!(`ADn+Gp@F`YaKK{82LdZ)3a~!a4NheziQz zweWnYxlRV>L2M?iN;hX$er++yp}BJQ$t62z9K3Hpgs$#DO zxxZmOf^Ds3x6pRoctt@$vD^krEdKsI65zfu zl!s*GrW&(qiLvYj1#o71IyxX%SJ#lRTb?L_AHT?>`Tc^4y{0z}+J5ZbRHa{EbthFBy_J$;UU-OUmi6`ZrD0&;VEDXg4t5@mfxf=J!;X!h zpcnl2$3rU{8-{G9U`QcI5KX|Nx9i4L1NajGk6nku*1){UZl{vx+#Cwd4G*W*m8Y8oF$gKMb0DFkxs1KV!gMA&+K{YfP{p!y!0e6bnYKHGBPsV zaJ9t3%SNOxy?!|O6x4`2dep$U++DS+29Y7w!S{ZmfMNa<4$btW$h;pjHf&A zCvTV2Uh0v>#VFuw4FwVUR8)D0pt6gDCu4hoo%Qu4;@w1KY~Cf)Bpv!eQnXNe%#MQ& z_}*?-4}-o=VIB3#V2jlXZd8%}2Gg3QB%+3#bhq6f=-z7~<3Jpc(R9*No;zGKUX0PCQ_t6wk}YL&#M>gyZMRa*inyLF;a zx!hsr0|CDaT6uYSesQtH$B!@5J)ICA&xD@F9MWDhGFJ0tbR@(!A89B^uDZU~I@=!@ z!gz&=*%KvVAsjKqbFt`tqzmLOPWM%DAVRPzm1;!-Y1Y_fckf2W{;{&<>rbqu!ChpVOT zm3D71ZxM@pr~H&OTOrJKfAv)UsBf0TG2=C-Q#9>9&~28|5O@v66!jkDaNa#m6?dJ2 zouFl{Zu+uqG5SXu04@MDE-QoL&|zNM6a?&-+>Gr&3CROX)!ktOs_|ed%R}oj@TlmX z9x06`w^zW`)76%9--J*PsQ^yrJX+=5I5-y)0yncIA}>Fjr0swB#@04^_3@6l6|fM7 zMU%a~y>>*VMu6TmqoZaJp?*HFSNy#*X=PzSujR4=qo|~`dc4x2KUp*~1X0%cA?NAI z%LBP}x?hN-{aV+!#mKxjj*^jfV7vw(%6tWF_S?gxh0bD@-(>=nwOnBUd*}dW9qb7w zDK5U;e0M13<@K0!_>ng~G00-E5f_NXZvg7{!5BTcO;)I?1N!aG2Mos2TI(0=d@XhLusf87ZC? z-n#LmOB{P5xA&5VmZOMTcOaR=#KNMow&LCQUn$njpteU*OuHa7Bm!s1F(nIh7SIy;BeK*DhWG`<>iv;X*l%QJwKb@~%1U-<8zAj~Uj|){e@QbkUi)uJ zz#o#~f;ArCMPV-|DcLQa_~%M0^ufPBN8x(D?q3&zg`(%X;ZXb$gZ`0lY-y!{4{Sd1 zJb!KyXG1GxVrl94&;OTg0PO{M{Y&csOHsf}D(J5@^wL-@)H5V8MF6)ki?ajR%V4?y zUoOn^_p(EES)|Ik=H~DL7+pe-`bSz~{+}lp@liw+h*O2XMV^=XIpE`x)$==XXTksa z+lM>b+ezE3fHnr5!edVbOpIfslY}h$tvDLA$g=k1#((|;LqZer^t>VH5X-55L@m~T z_UaIKY~CR_w)z62+9ac$)mA~`MRoJd#W1h#YwnmD2(Q(%dQW+yhlT5P5euOhf0d7V zP^a+?@~;IKYVl{szvuTwODa*W|N3AsnheOqkYa5%ZD zB=LKKNQzm{o{kjhsh+2dY3v?RM@j}KiQ~+NwMVBcS9j9W1c>^uoLU_uc91?ue^whm zc>oG0cy|?pF@RssGo^k=9=93`!ZpnMm*gku_c>wE>o-f$u~;jcwU;5^t}v~LN@dzZ___3xTJ|V+ ziT#KOW^HLpvcWO^?IOn18whFf2^j1b4nzUr#bh|)yt?T!e07(p*hO9}YRy~C_DLI+ zkJJCV)F&v_-=6vQ6-Ce68f8N!HJn4nK%}0M5j44~X@Q8WzL1z-Uk38sWH>5=IWD|I zipS4LKw^N*G#9gWs7M}>z){Puc-*98sj)5fsfm$+jHK&(OOgnY=UF{*pb60qS zd<-?esWiV>H+~?*ufg#n)SRQtv`fs&VzS(y%QAn&!(MC*;Fx7KLH$NjRENw9I-c|4 zx5ORCM{ZB31cs}+vSjKx*|lkeKS_~}W*~G zv+_6f(zR#_eo*CZUucOtE}`&!Mj-==PT~nt4oKg5mGf}>AX&)xpNV~Y4iXkNMbvH9 zP^T#0GI^v481;KUd`rGyMk4ka_Zj7C>FVN0pjz8iusuFf8l^8|;qPIxfhe=BX>4N} zFV8uCouROid7;neq# zdGt$=waqFQF?i!+>A0^B-gx2148DkMh^><53l#t`huv}VI>^!_QqmJ_2RhF>4ij*cwUouq{0__tKJq*|bt-R4`c zw^6WUlVmXp5#2cIYhu1e&1bzQS+5s67#BPC$dP-M*uWbny2S&9+XPTW#W z9BO^eUT?ODwv?s-KgIpDhY(xyK7G_qG5S+HBuMxnf|xZCvdVizcY}1I8D>9P*rTmc%0w2l?G%YnvzX87H+-V@GS-8 zW7-)_qVh^drTpg$28iNog}6j|Pw_!=zIq}(+i`1368q5FM#8qQP*T5}VL(#-9?;2= zk0=}U^i^nkqj6ali^!G*;k0q!?LIV-g@#A&Q9cwsyA)mTOkP5Nf_SSuPr5$%;=DT_73Bd0idG;>FtlgO zXq|t*_IVJm5?V1!T_v~<ooX!dXc_>UqlXO$LOoAjc6 zY(ru_!&gd5x+i=v*r7n1%Ytn<@2B_PmKjQ*@!I92nqBjWDaZO2TQmBorL?C%BLcgy zm+g@WWS`VS!YU2Ewao~xNLFt9=D)>McQBq>M@b>sp zv}4DOz#@WmPz*=i@#duGmz7CX>gVTb*`U4MYK3m2yN(zC>E7>{-C>5^ndu6JXdey2 zIG?Q0KF$krzK2}X4@{$uKGFERteewyXNv)x5&~BE84F5Zmd*6gJU9xMO()5ji@D+0 zmRpaho<1E%(D-VNgk>Ee*>yJ&+IA6z#;OGLquyn@6=5RSj^JF-k8gCZ0hzz@V?|9c zc%P`C0!iS0iDT=1C+6}hxwi`{o)Omw#qu~4#U++FlolLYehAiEOk8R+yX*WIbnr&Hda`p$` z-LJnrcW_=CspalZ=IArh0{DM>d5$DRt68SbLsHAHliNGAi|h@%2v~Q=XNcp>mFExL z`c*g!Q4$q1i_(RyI+-KEkveMmv$Zt@Xj9@I&sMtQi?#gY)LE^b2bD~N&9(LQH)9&q z87JlD4sD@2zbruz(~Rbs_sxKj0};)yy`$3ZaIEb3QYMYoHF$U@C}>a;hdGwsWJzZ? zv8Ms4RGC*lXxOn5OSzpm&~?;hGPtS!InOoBh4?|Y#tN^{H9p;9v}9GlHIFPJ7{SLT;eamd96V7_{XuV#I@QrPl{oRKt{246}iJk1Y^q?Wg-mPDCSV85GT)oOElW`ezDXI15O0PgXnIfn7XS4KHT8 zU3-|)?)H_X?rv@vYPTkO!rerXzJBo#R^a`2FTjRzOMA=oK0{gQ-Zlj0#8U7DlU0@L zRpZ@Q%WC^13IF9cVl#whpDF9~fTJxK?zIbi3oW)rhs)?XL`m~fvU>&UqnU;>L4smL zuxr#eL|i*#U@>V$3=hlWo8KdPovj){*I=%+jxct!ra2QEEj0QK;UAQ`tsDr3vk#7%Fk)j`(JN7quw=#?`RuH?pH)Su)#S7WW;6@d%#sWVR( zuV6VC!CdQks>K{5@G+k%X&W0GThV{qbrCoWcS9eH+_shRoG$!p?XQ8(1_I5x`g%N| z%HiV=@oU^}Gn@(NA9&Sqzlt5@3A0O`p|jZN8ug$gI;B8xjpO35r$+s!ks$kIOEJbK zkMFD#yVmBhG0>VJ?(Fy@IqdXC{2gK%WoN{~FD_7mgW@ z-aoe*mreF~&Jt2)8h!!w_b%&Gk=^MQiKNA%Lgj(ah$k8H(AMDl?ylKb8f(GiD>Cb^{NFe7i3(|vThNctYp1+af~%F@t0V9VXM~)d&#WP`-5!G;1_2A~k^r?iup;Rj zaRnhjt6SXj>HfeU-+G}C?nTgGhpJb!o>$o8!-^>lWXabF^gWn zQAx)ZFm%)x7|i*KwmVDhCL6(Fc;u0u%rmSVlYwrv_671{z9Hka448nDQ&W244B7En z`n+4qTaX~j?U$`}m4O^QJ$uhyIlEQXFUtFDKM)8aXwb%7ySux7iHV7kAdwOPm1dN@ z7TaZQJ)ZbJ`r^@HJc)_`3ILI-X~Z3?S=>c@M`9u(1y@R{P^S;h6(8(8^rlE~o zaZ!R)l8OIF6(7hEFsKR#bKLfQx`(+apPv;wMsSNT-a9znUN%qo%Nm#D`RpUR2q*qv z0`VL!#+KUmVD}dP2>WmCHU9?A=W#{+nz0KU2rKZZ>PZYf$uTI7eDASJ-Sz|DI9A{z z^Is61+F*%rTRtAZT(8GNUjR96cgl>+<#WozmkJf+2 ziOZ;W3E#2wG}TzY3^ZiPIJz2usSW!^?K*a1%grV}I|qlV%+<+>h=Bpc{QNwHG=5C! zeBkCmU{84b;?04i(=7vZ-ks>%F9DFPC*=J|vDUJ*IjONfa#9QYBXVxT z`sDZ_QzxwyB-H1#o~TE>HsUUk4P^b+Ov(FTTX|P}{0+UemOQKi3!Ty*CzdAT-931> zSm)!S7tA(dtS-mmIo^Isk-}$b+8QdlG2tHTxr){b`Lb{WMS`vSA526rwegc&e2oqE5>q zUSXDtme)QX0c~OY!$oE0z5^3VXcHF5=PS)2M0SLsvNi55PsnV74xdeVD-$Q|Y^$d{ zEnC@xFpA!RgcjHvo#7Lq;e9rQ+w4#wBO+IX*Xh<`IRRC9{eEgtTzrthp#&z%lLeWN zm7ie#G-G+Qr}ouXs>xyLCmT$_!4rE%f33+ z4Rn#QXFXnCpiW{c95?C+=LgAwP*QPnmhsOqe~}OPL|F|KWaQ4(&Y+;O2~eBC>kn0d zD$4Qn8H&4n$e~*=BA4>nfb=3`pxUcjI}`HtN+613UUY_^3-*YeDX5_ydy0b)<)}06 zaNljdx@`XAtKgU@lJ|Ub!F!jZ_nahh>Mx4VSLA)l;8;@J%WEp^UE*3`F~XaJa=Wh^ z^RV2*jmV}~60SjNZWg5q4K0hIQ#}aJ+uJFE?P+MH>BV2oDTK@rj0X;L$ZHkp& zFaJg5l9UJnaB-MRUN528fCsTJIwolf{Cd)$$KGDArVA zoj34K48IckOVLTJ%#&*yf%Wf||E}d{VS_UpEAOKwCa)iFBl45zxi3R4=zf=lh8?Le zBnqq-B?wcL?~TB-AnS~m`2H`!{)hR8N7+B3z6`R1-h}s^aZZWCKGz6;2q#IK4D04{B1{>)$ z%7JsY9bwcC1Ww`){jJueB!UZdA#n(w#jZz>=XA@io9h(AOoa)Dg})(awcUebHrz1p zauf~ToF0ge?zJOuTNFv)Z7VSiB58BxC9Chq5|%wq3+k}13A)TV>RhysYi-|kBA@>w z=Y=l>3cw5L#G8f;IgGClav(jmSs3I#fj*U>AlQDAJUqV}^xgbwMPuhE1qq^&Pd!A8 zUmz9x=i6W$OjVEeyn1%m_XjI){=D7%pH1MOg38}4`s>?Q6b_D#x;8e$7kt3s#g~Y2 zdLJa2+#fdr3_Y2ZYU2MU&^a1+v1x)KIAct7(RnBi`GW1v!5|%!!6sk(lT+5xj@n&Jb$`2p{dGGw1>X^8*QFzop#`E+#r2y){HV?BLDSNJi z$$@2M%;JLQ!y#b>g*~lA_igWPs7}Gt$R0^n1jygE47dGe#7AGSiwJ6d3>r5?IqYyo zVXq@+%2|U%%XEbtX3vrt5hUWPWU_#`2USAA!Ka3ReD^C(@Fu|0M94+!3YhJWEksJL ztumWW(x)`kIUV3d5eguHl%Qbtwv?dWEot~Ls8KJ9k#isI6EXp&Ti5js&x!&A3g%8M z8W0-U*vSkcan{OSNYtZ*ou~A$QX&a|CTBc)!_Z$7%g9!=w>=yJfC+ZGi0FN)tr~?d znKEjbfTAWfs>V(j^*vJl;L*+m&pa!PF_5etGwfW=AuLoNZUc<1*s*dd2nv47stP6~ zsE&P56{MH*KkKP36B6SIPC%WLRT^~b-Lqnn!|y)qVCjjPdzY3~ju*CVXX1t=$4A$f z6;hJWf^DgipcdlzsaJke-Amuga|IG0`&IZPpw=SQpJ+dw7#LH9tVt(+*BO2*bip2& zBMzqTh1p7LTB8I2H8wg`?CGi9Oipg@n_ADi>iDX?KrTHl_?D^N{)8*|U7_s&y|i=J zM0qv~!ZX9Ez*T{`0|x=o)pVk)c3(T^AE@w8^4^B}k);7j1nBh^hNCBSHocXDuU6iz zO8$Bk&Q}vVsrvd}(CIT8o#%b?hi!*t95aM>(Dzwg4viczlz#O9T+VtcUwR`&bI3{M zFa;O7dKN?r0VC7pSP+0Gg2qcL$ZBOQ{W(+Xqu%^vQl9A&q38*50Y&M6{9-%!@?7?x zcE78CrsZAeJFnsx+|?);`{|S~rcV5j!Pe7X!WjI@u+(CmG3JWA)HPd2ISN0!l!vG` zgWp%Rl~(n<1ye~5?UCOA9cg%K3YM6d*!7RNNg}Qp#DCV1itOK`Civ2CEaxRZ+SxG3 zi5*F}Y7#09c?k;DwRP#xNv8JeMwB}r7GHx{W;MbF*E0Dq>^L(hcmXREr?Xg_lT(5G05&%9D9jV>q0AP7L6 zg^DeWket0N$1mM)aiSu#v&yn(u-OkcosL~uSx$I@=DHgR_y@NL`?Q?Y87%tc(W0c& zF4yN041}eCHNdaomU+1NF4*51;Gc1(jcUqnir{!Uk?=8=cBO`3bb@sk$$Niz*ey4eficdfNLw^c-ulb;(05Q#k)KX|+}La`fjWQ34i~D! zE8OYZO+Z`bDdTvR_uq*y`nq^c3-;2{Vg89_4;x#lj6XS|Qk|oSz!4!b@+}WF1T!>T zR#q10q)M1#v2LRsP<}8eKVqjy$7heSCGw%#%AJ{ZP&XnsTD;<(v_UjHn&Z@@X+ z)zTo|X!QB-z!J+WsBH-=!H9^cmBod`=qgMjLV2a&i%Za;_6 zLHy8fAsFQOFUm?_&OCH#f2v+pl&TKOyuI;x7kw$4vv7MuZJ0l87YvLaZg9 z9)JMgjt(o=V9K~m2s5BQATf-mt6|aaLISnU zbI6g+Bwd=kOIpdO#l(4PuQH!n-Z9--&|jH{lvh^siD58ohmylibH0oM@1lknDV!hk zNFdrLW#u(rSRxy6RngwK`5F>vClcvr@>-cz&h$Ek-yBHUnzp9gMj^LEJ+7b&E(MHk zdHmTv~Jt8o%dx)tUHiP1T4`U365l719y%O@%`Q-CZiC(av9hETAOyk}>A#Y9RHyn!(;|<#W z5x85f^wG0m>7DQ`~I)m+`A+BE1>3r148^|Ka8%Vl!3V+xVUHObP+A_i%D7M< z+MTS(Wt?$L)e&t(>gnlWU{5)d4mZ_5g2jnuTm&SutHjmmU1$SF9zc>PMZec2P+Vkw zf#lZVjRX71m;;k~Q*8Q$fn*K&<~0+#hH8}mRX3@9Xd2c+d+!v0>d-M0>?4I3m^!S z{{`=6{$L%-N@zV^4J`(mn2stMKovdaq;iqe?cV%Aum=qUHgVAQ4wMqovI;qxJuaNq z%DmTuo_!|rJnVN?s5#Vil1VLbG4O#fa+q#{r>l3O=PEY$hg2^JPfkafUTO2aMAx1d zrGD|8n93*H+ZdkUtuAxm*WY_(2gtp*oa__O^>RV6zdD^9&l&uD$*8hOqk8CJ~RgiagftJV74_LU}; z(KOBIIjjWtf>pFp^Z--iJDUZqtB4EW3vLXqCtyRJD zFIlimSZE*k$i)F3RPZ+sI{AOWgZ}AvYbR;T{S);%=1_${=lGK?TwhB+;Uqk7To1?L z*KNr85xi!xQGRDQ+@I9-6wl8`X=rf60E+J4_p(O#5-05xt#^^-2?=Aq0UXvdKaZqA zq**iXQIcZtCtC21dsan-=7IM4FuT;Kz1zsn@+0_ALb`q}y!!u#BkS)R;#sZp!6s`M zkAbV_t2SBgX#4gr8Qeg%ffXghHZ+>ls>;540%%o}*uT`o%F3XKhKnw6)V}MYsrDI* z9uv+UMQ!jBj6pFoe(WCfZDHZrt?>$h1T{rI-1vd#Zqh%mDe@~xof3rB4B2Fx|cfL3TCxlJsP_eWVkRS!e`K;0XF?M7bOI*szF{| z-azh@{Q6Z$u?`+BDp*Wl)-H0^)j$Y!oIuXUW?nuwWdDt;jqcj?-YG6OAhwUB?WKn@ zwn`}im;?CD9_LhOSeRdQbo9&zV1|DRp=v14bt+d=Q&ix^SinnBPKIu1csuRA zv*SPbb~zao9}5TjOQ?Zqt!b}3=TQ*n#Ks6Y7!uWf_|toodNIJYl=q#fgYfM%gc@E)0;xyL0S?Ivw?B{s z<;}No^#39KX8uXPB=AV@0lnka92&^A@mvweE&{8aTe*(2==S%CakrIV(!S-Jymn?w zUr%JrM%<>A+J#c&W2B$q;VrvqiQvGg&Wm;T4^;u$U>+ty0praLcs=vKm%o9Uea7U9-aB#scLEov0IQkWnYptq(>e(jH~mKZF!&%eKgbDom(yIN!=ba&i-v zGz7xe8Pmyvv9nwjwAikkxLZUoDSXK=Rz4wxf19#Jj?l`|-xd}5DEvMCfhYA(jpH>t z!LJjE`~__U2{&S(>xIAtIT&2N`wzV|P1|9>noJOq;s9xfN4{*{AEd9(jdt_fG8wLD`OFd+6U%L(N9xs!l$$=2aHl2gUbwvR9x zQ(tmaZ9j4b^qS4mxM6@$q32A``TD%|2?8S5z8K3P{+1mQktrb^4xPP(3mX$^Z|kDT z3wncmL9t#&4Q~EA#QEs9Y6nQmSWKMk0mc8JB6tII2AfLpbD5-;@%}#w;oU)lIxO=O zO|>Pq4(APq3o{qBmb!HSwfk%L)KHD2w$q*k(nZ(fywWQ@ zpYsuaB$A@P#Y-kyJ4;vFXGjj!$@3?Jy>TZg&tJH{brk)qb=0xP<%3n8x-Hgcy7gNX zsGb!M$cgo3fq}n6@YgtdjDAN)N8tA$eQ$h7l!1@sroNC)Ojqpj7f1?!&%y)baHyk@ zGgKgOSz)nX*Jg?F9%W|XhdF!s+~OKHKjgz(BD^JEa+q4O-~my*XjA@)kRBtakq+qI z?r|?7&anDx?7_l!91*O!X^t(?%t+jeI*0e(V9FHUo^6z^!l~U_gx*5d^N5zwWkw_0 zA6$l*`wNMBXFb^P(CY;3vLO*C0 z6aX!|s!dL0%JZO1s-m({@IgkidiS&ALE{fQA_&mnnoc#W6G6>+!%p>RPsNI(5||I5 zp{!hI!_y}QIAgMRA3WqX>H8gJg#JwA^QBz7Cn~p%dR9llE!)(0b&~69&Jr>I4HJ6{ z1r6+SojrF}J=r#zP*VLyDyDipd{;;p)-#RpNsC?+fU!6eydru{|0`n@)KK-4_Vwfgkn=m~1! z&*uQ)k4FdLH}Ei1i=xKoG!^zN z=Z2@>Eh=x83bp4V11FG(52rh zd9SHv$oRR*g!=eVwT~U*k)EJ(#F>W_U)5Ey|Ktx&EA$IIlPwUVdGu#9t)9dVNQN|3 zwfOSUX`gfO3Q(oZ0ZDr8GV~RpsAQ+4s9s>;%ff~?QV!d^x_9-8ALYq}PwU4qzrgLj zK%C=HROMDeycm>EnGc;K-mByiflEcu!+I`I2&*_{2{O+}qEuAYci8n_kb-cW+9 zWjz}HU4o4j^c=-?@PNPZ9~X1+_hk4=)EAXS=eE2tUda@OCE%U0S0kW1*wV_p?Z8L=cA^(U!PpPK4}h{A5>lbOn2M?+O3A1rc3lVAQnYbQKfIJ?_VYQH{=ChKUO*i;rV>$U!JqF(CoBZrV8zN zmFG9$9H~HvTM6%usz2#XvM^QFKhvRVUb_a8)0bXf301v@hk%V!9l|M6SOjg|d_2L^eE7Rq&4|7*6^BIM2<$$wo{+-GP ztm-q~w|u^P-|Ca^u{%eVW%z2er~c7(S+M5{Z!Tqh#-{ zx`V!+l442n|8lfryW>mXagH^+^iob!520fuDZzPTap911cx^c|9C^koH#J_P+L9hnt86U&3Q@Lm4+NRKOjOM|x)*aW+q_7UtAGPZj*^ zr>rK{!0{=i1AHC+)mgt=xV|OF$2zgsuxp-rg1k#6OAQVN2F@Y0n((?l6fT8p&qvss z_a8TXH$^Z!n2tU6J1J{?ojB5?g&4ylDo^3n=E3r{wxEPcNnF}$6ec}8e-CUrxPFQ& z(ZX-QZ?{zMFQV*^i+~wl5!#9Dz>>$mhQqe45+r@$?6BQoRF9WlDMzrJD?1xrxIS2p zk#cKxih$|s7?%8!MTdMsIwD43y9@vG(Cx-ei$2dNB@@NT>K;Cl%o@|1b_at|@^L*? z`&DZW58FC5T+a0O8dI~}z?_(wV~k>+cTJNs`Y?m%@{G@8g$nE~kDRsA(=?hZ>h*$V zSW*nWkpT{8-`6py){KYBXjG7}Z_@<#LKLJm=mJY=JS}^ErCXX(=bg+eMDXog{Nu#r z3Z$Eem~g0F=JiW+jd7jP;mFp~b{9{I{+kraeaL-!Y2^==m+YvKgnJX7Z%2cT<~mIo z1T9F6HHqm6QTC^cztg5g(gNRnH^WcSpxm3l)#S}|)`*G2k^Y$-rPv$pAPUxMMT}F4 zD@hiXCVCW~G96V$8U()SV)Y)NF#QA2h{oG@l{E3b)1jz@@<-;-oqg7(aM*u}qI?81gN8AR{rj*VBvOVBp)=fzyi6MJ5l4D=X`^pCJ{G z?Y&G2$hG7~49qHK&Hcrbcn#hnQ|2lic~=sT_GK}K!_P{8>0@BO6IUgmDx{cdoV4I= zcy%DMQZxl8(ckiRin~$ia}8Xf%!CI+=N^x_`Ot*ELOs+VsM~ zDCc^VS?300V!9ALBLh_xfA8Jrxr=#80f;5?@^JKpJ?)(JVw)?=00Vy~7Z>YKOl#fo z>98nGlqusttr@daXOykHo;k!Tod)opDDtU2sjj!aJjT0SV;MA~xHbwrgi2rIfjRl9 z37@YgIOS8!Z4AD{{)>0FAr%ZL^HlT!q-y?ka*H2wvTzj?i_!!w`LzZjt4DMe4$)E& zd>FwhHZ%x6Brn$9>lNl&*MfO^F^FN@`6FYpAL((j>ih=L3(K^c_FJRMj7cR@aT$Y$ zp9>~9^-2JT{JDA3=J^i%ee_i{8L&|Nx!p7V3X`nCe>(ZojnTV^>|clYtOjuWeVXL@ zjkFJRA(58wAdDTIpN-7fd6v5!rQfio9bMq>BR^aRM(ea)hTx<=st!H_VGylvc!>*g z#`xG$e&Zji=4gy+vUT3PH3AD2^akuDJK`drvcwp|9Xs*cq)n|v`Z$*ms~jH@-=-bF z$DKVwTFd3)*V8Ub5^^rzJ$)VpA*~!NA4OD#^_#S=(eo+I;|CnA@BFIry|>ovxU9@H z3+JbVBM)az&LPEJc61>DV6XL;5pZ?>EsgAkH;4E{5DR~U@8Ct!iz^DrmwvU@XC~P- zA!0Ar`!uQ(jA>fie}(=ciMZT_`3$HdeFp1SkDQqUX_Ilr8^H7!<%<7)l&9>h=GUrwlucNzHmpGdxrPz^Ytyr)>T|CPq zqbMr(ZW+>ab(85aJZ~;N!uLjtd&jT)^gg$sf#lNT^gA;!Bz*c$7*hh`%HPNiYl>60 zY6uU^d8R@Jnf2W^y&%G|hG-wQagkGNA}Wzq;R|Se?;Y{_PlogbPJPMT`9%kg-ffcI ztqWCWWB!0#p&C4lt7ET(%iWHwoWQ>*$l5bARvYZOy*tWS_yLjR3EC1IVtLo9A-cWR z9)a@g%SIA0Rk?@cz`-{>X`kWi)vdJ4Zk-2L`^*%mNC86&qGzneL|_Hg4_OSlhYH zp)pF92*1bvMd-P13^(=pX=o-A@RTQlVjQTV$@1vx=c${hOVz1hXU#cqS>OGNzD-FU zLNT$E;qPZ>;OUL6gz(%Y3f^r#6!KSZWDE-d@4Z*Qt!1<{_l%_)-;p&;GQi{&63S8VkACRes%{f%(G}cVSGAgU@-c zIwX0)iRos^?zlS)cjD>#*^H1#e|WCZ#6)yR@?~b#BVkyG$~_wAUjHGW-}B*%v7i1`Rd@IBLGCG*OyMFx6w;i*#w`_5{zu$Q&H~K!Kf_P$%$$x`* zHLZ$%7WiI<^P=}@w9$7}Svy#lE$~#y>k|s+229l2vWvzmCwX0=;*f8>Z5qP!6VQTb zkU;af*O&Xa6OB3}u;pd7zO0q@>`J6I$XqQ5 ztwcLeE;G|06eNm!ak3LgWEkiv&3welI`yM=t6I!FJ-{UR)z7TzuWpd+@x@x}qu@>U zmAp1C`Q@3Lk!~7f2Oh@A@!~WDi_}9@)j#5D(p(bG{SrC6=eQU*8eMj%)HgThF z6unv~ZWHWL47}XL-st9kTjNgMP;Z&@_<|ICl{P|N%b(4A3-t&I?GDP?zJ~JZ*YYV` zDwb=lhj8VN9Q02o7K*{A!Yr-&xVop-(;^RVDLf1Loo5($fOY99%?(qtj?>Tq1(@TsQ0n;uUss=M|E*bsa0F!sTVYN8Y*wndpznM3V_~$JvgE$2;l*aly?;Jz{K6|$TKNdMF`h#&7l)LpXP5Nbip0}rJ0 zc-6;39-MUQ*dYOba#kyS;yzJmQEDO;Iihib{p@X3k+i4?h z&+}~NT4bn#@5A1QKWV^;Sn^j!gd@r-g>s1E_f*v6a-uG!3mCsRj2NNRa5{D+%tz8j zH~l7%WFHTEQ1J+Q^hGPh1Z7;8hKmT*RCUMC|_@0;1-KBmN>_={( zMOGY;sPgfp0_BH{1BY4R15zjVIrY;4Io>;U7z?(o;!kr(zBC0T7qLj*gx@+4UyvYDbA7o2C}$_+5=_zvUMpUWqtT#oadC@O%HMLlHN+3V zf51e+s3ehh)upV5a1RJd@n>CsyO3fw-YVMWS1RoifJR<%QJh#-^YmRZJ|9Hhs z5GcD{6ZAoA&-0tV{|v$bl7sup0n!Fv!Neu6>~9)uxHub?`Qd?o&;KPd7vE3N1in4M z7<{)M^nrTbUFP;C&Ng^HomI|Kyqo72M680YrI`VB)YXJj6^iG>FvFq&v27(jWxsmJ zMQ~b2+}xFL8eB@T0MOz~*TNzM=)wY3<~|VIv>$pmwS1B4_40BZ5)M?cp<`oX8zp8G z*F>F1artaUC`3wz z7dD>$m)A(@l(ShRKL`XWS}A*TAD^FXv2n6yiYEVOL?@sdy!&%F@Rxscwc_6kMqu>c zyFSG0|F_BR-`9Bl$HKwvmDTblh|a2i!T3R`7EYNp;@Hv!%FSFXDk9!X?C?2anm!TUTD?(Rz%9m{K0_N!)zi8Bfgo?l9v^!>G$ z5q1ytb7RpNv}+(>u>f%pgeGop`Y+MJ$Sd5g5d}H)3nju2gb1v zFmHgCJOq(hqe^nU6zcoV2RpjUWSp-SGIkA-(mu;uHnvUoNhyq2EHbhj5L3X`A^a!} z8>CE=Qk|!MnJ+QpMDVoM`NF{57o?Lkv8#BUzxq;=mA>LXpYH(%N#MO5Rm3==`RoGw z^i$Tg0t>C~svX7$Yu^1VP+CvjDf4mD0`Dg*cJcv2luyDTdfdB+#BI_#cn%3W^ES01 zNHH^`mZzTm+@v!41#3BKB+4`>ipibZNDTBZ4|~j+ESvkG{d&Q?TesG6JbNjf*+6+n33ZB_ES0O1EXezHq6 zKDHj~-?g%9cZ{0W;nNw?y+OsFLsXXf@nhCQ%dCOEsbNOqa(wq#b$mx;KDqJ ztCYC-!d}uWo>4KOOp<*|sHhQBhes>n`5?Cy{N-e$Crl(+*#@tfZL_nOrO8qb?;eAr z@<_7uUteLRSK`BSpxqxc4-Wys+QS!*g-2AC1D{&ss&KniY$gn`CUerbf zJJgfUNZS#6T=jN3R09(4-!{LQO(Iu*EpL4?AuLS&%kRXaV4B)nP@|s>g?(}?{efUv zRAMvgW@3z|UcPJtF1J&{fSdx`5gcd%`zGRvLsJ2Ipg>p}E=uvQH_0fZJv*Rgu3KX% zZtS5-rj%{>qoI$Ui(2-IEoeJ4K;E5By-Z%a#I*W5UeGAsJbPN~%JoESgrDB>o`!(Z zKwE#MYH8$&a?5V4!^wW4-0bu_Ki!}O3%Q{~+yCX)vdU--K4jcmx#DaT*G$?)nxGD6_%WB4^xn$&wjC)R~FvFe41cu zErKZY0pg;fa0k=H%ysMH%Vr#c8!gIn0sC#nr*DH=aC#nE-vrB9z=LhRz?&c+lmHZ1 zMR=TJo_alwx^q;;q-*S}r0A`#eBJC(z4Uu(eX z_cX}J8ORq~>ROwM;K$ba#^p1acCBgroN-ejsN>dgVft1?mPz9>4-t8-r>To}%(rcvmeQ8Qu%UCdrqzgZ&>`1ICn4Uf^ub$`rY(6LN1qekU> z>`2H1$8bW%=Xh>LH}C)P!3HK7W+B#u%$5KXne zm5$s6UL50o^Ef{ttzlwi+W`qqmy{oJOpJ%f{u5z&I2U+2564potK4_#j~!dwz8}p@ z6+G@x{lls`qo2ZmpBWZ|!a=U(qF`UPl!E6qJdjI0N9I!Zj-yrWn4QG|qSIIi>LyS* z|Fg@~R6GkA+Oj(KNBktL_0;!_|5yN4{^vmC!?m<@li2 zIf0EJ{yV%od%f1|#rI!4l-~X8WMA{&NPoboQpAuS%fXzGK5`3Jh)N^yH0t{{i;^vS zX<%?Q<0SmA6Y09XqG*rc!@I(ikM;Da^&}!Vu)1envVOp8r35>^jqVEVvlY9zTmgo= z#~O4YqK6R1ou5m|g{AoIjXH&M%bsG5axc-pBChWCD+Sys^FMww>*Rgs(DnuaT9qg9u%l?dP6^eB|s%2uF?RT919hm%UCv;fnOW9#vF`tUq z$fCXK!uWF@6OR_cGwSpHFGul}CP^6$8nRu_4bW|5!7(L`{q8G4Y#CqLry*$kMhP5jcs8CwRw=(ggZ za&^1(lJC$dYB8%(bzC3CiWu(rA$i3ecFt-}2Y=I@Po(Q)!a;X1m(7T++)c=Hfbje$ zNQVD2BeEs`=aBnvjZmzZ=+Tvxk?ZU0!IeDgKs^AJTqd^0G7i+KffCH>Oo3sG1(j(o zn2?4vPqnNj>C&Tt`1bjm%K*kuT1tv`t6IC8(=Ag+j22)Rw=K>xNhf!)+9#RqvavET zQv6Atj(SY8!n-~E+N}#x9UCZD<_YSLjfID$t%d`*5)ebqT}F0YiKfy@hCft0WnEL= zNhH(_Pbs($V}dlU2aE2{km*w9!e{j6xo#3wVLUF+E%{62;=+tR^)%*06e1>N6D_1m zpR$=!*zIqM(`z?iIEXKX-XU)$N!|&myx`%Oc{a!ffj~4`+!KM` zwm`N2df7Nmv#x6Nrm#goW;~O-yml-dL&Y`&< zRCCPlrkaBz`9x+S?Mlx{p&Ih2!?VwuI6F85lXRXwXVj1=5Yx70=`qDsXNQfYB$Z6f z_3KXg>7DiB76Ju{%?l}i0IOoaS@qqr-w404jm{TsObh_-Svw~I3Hrq8m>3iQT&af4 z&n-h7%~sr!qx@fVf1x}Km)*j|qDM4)*B&q;_o6v-HPJ9nf`4trrFp=GJQzQE3k9D1 zt!O#nxgWgq)xV}XvXi(?`eLn~tdNt7@uumsT1)3z+v{}#He+Yd*t%wFYBIvvkUZ?L z3amTo$DJh5>0p887L<^|-8#f5wSh8iT9qB5h##I>`o0^jmgOKXI!Wz9$GSsqd2`GG zL^FK);Y^=`*s!Xxql0BaZXsqs8WPbQ&&_ZsAK}hE={{<= zN7uxPMFEem5d{7S!+I($)*7@>A2| zklk4vt;JBUoVy|I4Qy7JOR>Dumxwrq(7=`m;44#0OGRMs2K$nhf_wD`#Qp{3<1Z(= zUAJNC`HqFHS2V?fGtd1)DW|{%M}Ii>a8qw>yle=#MBVcL=FbZ z8H$oUMc3eA>gMn4?16SE3d_S@f_=(f%Y3ITRC9YoBF|m%8UKu2$JLHz?O@dMMRvEfwO- zM9L*y%h!8Tr~FQb8+Z7k_adWmzZ~jLj~RIE`iW#6h^r=t8Y5pAe=Tg!+}t5nYA&-+<48)sLLk*0i-XU8N$GHR7l2yYzFERq=*Ry3-2Z zu($Tlbm1myY=EiAim3^gPsR@}gupvf$BiNur7Utj>1H7xtzTS{H&h0EhW@tPn*S5~ zgSHuR*uL>c8fisMeYD>~=)&9htgHYSH&6d4mM(j7)tKY_>@HZGE!N%i<8`#$+UbI4 z`gufOi#{4AP}9RZ$)Ns%?BQ8SJ*VG-Ua^mNz#Q;SB73cz{E}^jRrE;*J>(5hee|~1 z?tE{IZd|+;2E!9d7a1@BU~cF1T@-#UD5OmmF;#G-)MNwO+t|r*;RbcCH#EN91)h2m zlj`Y4>S+bITmK!MKbF^*PKM3}n=V%`k1wk>TQv7j6hf#oLS69%>owI7b0=qY$WX%a z2{Eap!!chX`FWiIUElY-=%N`%vyuvv9m{kX7#1+L89_+Sp}+Hf&3R_p z*)}n1RN8A1ay6Ve9KQL74!1pqvGkT5qy=JTC9SnoLqTo?fBZX9Ib!p==n4I$&rfjp|XM{_u-9 z(<->4bg=w5O&w=&#V8BsR8mCJE(^Z$<BHrnoKcWlPnS1<q^tdQ$GoMihF-v^*DHn7pD5> zV=4|Q>|R%{81m-mVUSl=z|`sV;faPKg4YH718~Ybed@@m9(&UReBJRywluQqDQfFr1H(4ej}RWOt13~h?-xC19zD+Ka(dFM?Gx-csU4o@t6c9*&`rx}xlY)V`8YL-LkMJXOUw=l-M!`y zEO7d8Et^LunKR5od7#r;z%f?>`Tk>IeXU)EEdH;UJK0&jOV}_-4$aI6z9{=2$;Afp z*n`fVkQ!&De`YJH+tE+Jn}NIj@rTTZ?>P_0?3dTOCMe7N#M5;@+S5}=y|jJLo&YPi z+%P}3jgpX-I`&XSz=W31tiW&spBfjzWU)G{Dj%sTa6!j}5DRKDh!4{SQU2a*sIZ#O=zA&l z6A1gy5pCbSsrt|EflcrK!9DOV92n927YAyf3~2~o-fgI8OOt8D#anKM?I|=kxxtVd z#cvU|Wr`2QVovgxUqLtL7q1HrU60F|Ev zicixeOu<_Zs%11a7we-B*k=|lHD{IK39y~qIP{xNgkPEP#ItvNMXOrzwH)ZBN~`*; z?Uat;w1E9s*=@${foeARir;FbMM@)CX=J91=}XPL#JJ?X4+V-*@52j4E!{P}-%bRu zXLfykBG3e!_7l8d0w<5;0;et>X@~vVoNbMLM+9bv;ZLOzs8%E{d|U7VbAJnmNp2lM zmCjY$&U9kae7=^ILiUcRpque`u5NC9E|+sBMBcZ*dWtma<*#1l zc|L)%#*;=)?PAQYu-jxW&R{{WxBc(h^HMYc`_%an3J>b~hUaMRdDk4N4W=KXvc%V$ z-oG&V$4lskSpl%%fPl>}~d7#nS+S)1$)dNyHAO!tip z;_X)ETHk>}D*(QNYnjzgLvxn)ViwLJGgnum;pAo~nad>#JvP=q?Fp*fjA!Rbu3Ify zldNdW6L)-AZLi1DsoDFZOj*oVed`Lw4v5VG*dv*mPc83x$)0Ec5eEJ1A5mrtuxule zDci7rDdNGGh*#i!_iIU+4?^^{IGEFG_T>{u4|&+w3xb zLMbvM2Up|M-8NfZ8Ye$#YhU^DO)JTLRJ=t%*)hLrMojY>UTuSFrKWrV4jmx&c%FXu zfz<;)D`ozQDEG%3A10EQcch4B`bv6XFbk{(7{%rkmyi~qd$qKf!?Z1 zl?aVElgNHSSfHWL^=dGV+wy|TWxtp6V6dbF`OLn0{Uo}bG>KxRek^xG!$TO-URUaT znOXcGyi`BP72P9j5_adLwsEg*{vApE#lgkG&I?s z*!Dg*Q8seyVyV$d)!7E6H7@1wzI0^x(|;?1qYvH>`}3@}wst~XFQ9(?`n9>aZCfm9 zZYbQ-eX{&e3{1 zKgEKTu>re+-90siCgnXk`GCR_bBOHCchp$VEdc!F#vV*H_aE(7_FwF4b=8H1%>Rt< zNB+E*j2&};zQTKz_$jw#T3|Tuz@uO=u0&|dMXi3#oM#xZ-+>=8l*7jj5cGI{KyA#7 zw^H_QiqaI0-sJGb`mOp8Ac=#k`V&mrHsxkY!xk!MdO zwd9TEbKs9qi-h74ngd|!rFk+VQT!7s`Ft_1rkbU)z&c*dZW^N~#D65^c>|m2)%v2l zJ-s3TkGXms#sIZ4jQl`|ffi3wrhnan$2^apR0A379sgtAsffV!Fa4gk!!#VBsiA9s zl>s>a#A^QYei~X0tVKssR)Bv%LPdQNo*rLZ!5_M1N7_VP8)c~=yaRRsM{~6jCXUbriGc`p z=VuUSottYQSZ2n?{H+ZY>UBG$J&n_xIEOEhd)Wv|2WReE;+=t za53v^G~vV)y1;FwW92er-v>jvDMi9oMSL_(#U0K!wCFzff5cHOumiO8jGlMq;?u$@ z2F*1JcCjMd)-WD;=`O9Z8#*k8>^~K1v#~WXb`n9FhrWwLMD-yCs`vwYBmU(M=WRqL zdtY9Y*cIb)apj~RG#)O?AtnJ7ehj7}v3MW<$Ig9yiBV_V&JTL~AogD$UVzdvmv(+~VQ7y@>N3b-61RkuDxf zu7RIzFb1`pWBpZ0bkf{P`CxtU=gOhqBOVrN7Gh6IS3*kf?`TvDWnM(T9 zbA>C~xREWQa))}3+iVV5;BB#|F_OK#5s<2{o7fMhA9HXp8?x^}LOun#b;^)%`n#fU)b<}W-D zTfQ%faRJIsz5tBT`=jUL2nJeBf>TBOsbKDIn9$dD&rnla+D|Ypv*2eU1(QAo8;*Im zuATR-nHg?=>92Y95y-d5KTnTG~YzHKKICNf|p1L84jPGJRTWN2bb-zP~J0iHb!%NnDlF7_ zu!cMMi^`-91`goF{wP&R&*1%VtN7p2mzI&JM9?5)5I!e<@{#B}h`hnHT&6%Jqp&3_ z4gVaB4|OmgS>6KXhJw&-?fVL2u@|X?*mhvjXym%&7g~F1C~*~os;jJ?A?ulBRKcW% z7e8F%zZZ+K>oxBtRnH5E&nf?KHJ%CTpVqnPON{Y5GGWg7mRJi^G+;OzxA;s*4}Y{u zKOAK2;x=Qtp-(yO_lYcle*ZD=0pCN*LuOhIt}vyO!+kBCzth5|wascfH^7cpUFWrnip%m{CsuJXbGyaCJf#fkfaLV*N5Xf~2%p%~&hux8KXkdE z092)rdHSycUkJZdgldICrOaLrAdo3qM!4RdZ#qVX8dyOv6zbXAWTbgBkH8Picb@v; zuio^$gV<*un29gB3Sn|2Drp-V1)=i7#K{2SJJ%~S-iGK3ijaZa(s)_J8FL2ci&|D^ zZkg$ca)VY7%)uCK>pjxJ?j)LWlb^Ucw}d0-=d?GjnuRXwoH1EM#tc=%JlCn06X}(~ zI>0tXA)%q=27Rxg;;daFm776-RmrAD5MX#(U$1g)@ehTrno^3=lE>x&F$(>mL9wGV zDFyl6u4smahIlS#^>w*kbPTSk&JQC@;JnH~9Eq#UM)xm5qbIF#g}XK!or`eOPLsRr z0@Cv>8{q*o_8|4%-*VmsbKX&pIcLQsu0;}IW~E!;Ks^1}!h+;*<%s{)i^*_)BVeQD zhH!z4P6#wWm#+-nm=%-IGExX8Fa$<3vM-{F9v%L6S39k`pb-OXOg6(Xk#`~G*LwN4 zgJOUH@UyyvUR3lM`~EH9eCfGG#_zBLY#6Ek^rF$u{)b7;g!CUKwSOus|I`EjVX!Vw zPyPznIl6m#gtR#n|0;bq>Ijt~0Q;TqpQPaITY*OYS3f~8eeWy zC)7xL8chqN@p>$To5y5l;ZFQ2q1xQ>SKOTUnjQ%+9IEq+)o97HqWB1;OOaFAFbj9tYi2b}k zU*@8<&%&Vc$U)U6MdES6hptH5c!!9Y*5_ZAf0?yH7%N55vtd=|0)Zwed1yj9=rBBXq$8`0}nR?exaF!eqX{`=EJ4srGZhn`j=gKoBkCaNj`?CE&tp{A}L3-BQ+g{9PwagRQyj=tO^ z?keRi8%hOB`(TKK`6#QmE~&&tV}bp#VD|2kU>C0cB4m1^e`cBixdc!!+5;q7<;Vdj zaPiIDHFi0(QlfNw2zgy&C#LpHTlRD+t#*W}B;wuf=OXoC>M59gu#^@g?{OwbAsgRB zG06Vx7fq2oBupz+K}2T8b*)*#I+9DnLGvJ>f5tuUoCW|t9K{-?$BI_UX^JJvSJd4O z29xA~l_*_erzu?{?fSTW#DMFf8qQW!6RcoZV^2suGSQE$y#dmaExFsXX1kcnSE-$( z{Osu3f&%Cs;HrXdv@FLGZ0rbQS#3p@+Z4Q9d5AO_NOXoq!aQ3I3x*TYZ#BBIkc3gg zLRGwcE)pu;HRr5EWj4910)Qzwm5r~z>LvkHl3ox$-H+EbUlIwplCp~MZcVL${d>_M zcOr}^aIw6T3Hg}BByI@~EULt=x?{Vp@>qcWA_T!*Gb^LKxe<$Hc-9e4%#8)oYBxQ% zKq9uvUb74yUu9<;EP`Yx19VfxY7Krqz&sc`J11b$C|!#^(Lpq^KhzI62zHlhpL|QU z-vrM7dI6XyUzPTEMV(DjW*=o?>~yc2Jwm4jy*qzqPT_e2&QHMjXVY2JyS2N!)|Q%Ppdh*Y->R3aAm1lDs80D_NIUO>zdHdn zbmxZ$pGDR9CbfaCc{h_|B;-Jkc+={#*N|*rUPaoAEu=UL{!%~^J_ah$d}wrx308B( z@G4Agwh3-0qbTq8qr`(Co)WSC-6f>yrpI?PS2*$;mMEKlcunhx)~zH<9+$PIqh=d?L-P}J4kbRy3}HBjNF+_x+@U04PU5~%sd1esxnX@bM%g(jTKu;#h6FVTb?RB0Or zvL*>rg*r={fX;{hzBq*91b}zW*s4gkCzz^AN$c$zJ42LvM$`a8tL{kLRy)Fnc^nTG zCC#0om1ZKp$@#DPUSHLLX;elsLiVLc@E=y;wA7Vo*r?xr5-01GCqD_V_kZ$}bS7)| z>*PBGMpC6B;!w8sCx|*fx#xyiX&c@Te$4v~>GQG#AqYKXOg-}U1wDn#*lYt+P${A7 zC&!r8#L8WX`czyOQ=5F_PiiR{&nPrGWQGHtnF?k!%~0^Rew z1nk?%T~ysJm%Uw`t1c)ZP4K6VbYHHhzOncFq3cZvfao<`l(7r$)ZzxgaJu<(yxxnB z$jNWL3+3hG`tEx*62MAt%J?sXT-w15-)aaP1Jf2@tKyTQ&DgUcok;=ohs(2Cr zdzOU9p};#FC`rYQe8d7S==o>|b;Yaz3U3wK!I*JX!)5smkj_RBc13>&T(K_Mckl4< zv`|NM!ej(H2^j@?x}j_yW65pjFgPd&(;=@wE`G4>J{saLj!3RQl0VuI7Sg=TH#y^T zA>oW9b@BM2wg{!7trc|Pu>!qoO{GjsR#EpadrrI%^ItC=Lj0B?e!c!}%<rXtx=-`-auov^w@70LYD4HF z83Y#`ZW#{S?N?QRt6lT?|K8P}J9Wltft3E+*WR%D%@?=gHzisJII<9K=Nh$5t*o}o zdx#Jx1ksF8M#deb6S@N#HJxjc=~5UC{A_Uld&QkgLK-dh-+--fZBsqin#9sQ@0n1p;2mIzW_tg+PHrZxu8w2cdKN9!m*mA6(myS0oY)HHtX^6)?F2KgN&ol_p}xy}_t7Xf-L% z@5XzUxPzr4J^6X#((u+meD0jwb)4;hz|T_mG@@jVVDnwJsZmQaJsoKS*)c~z!w(-5 zB@VrN>8s7ke`GJ$Qq5_*G!mGcvY3qOz=(~rca}PK!BfDmnc`;#7X2l68OsPtDa^f2 zrdh8$NmRYr!B^v|Brff$ayVtKgu6Z5TkzD#q9q(k2y>Epqm0ca$y+0svTKSH|73p8 zl}_ChHq2go0xZC#pxTcDynXn+?D3ZYWoXVeC9r_)^6m~mSMPysvGsuMk94z50jV{b zF1qv+s<`dj8lkZP9KQS?^N!v_XY#o&?YBMR&{og(3U_|dEN8_|Bujmd;#H7eYQ))7 zuqWialHh-q9$av$A~9m#OfyRZLGocH3N4&7^TdjLpL| zVEHpW9H<6=GG=z(%zJ=4cG8$R7~4u;zU*+@I29v?-ffYUTIn{)y=?BGG7Z#03hekJ z`&FbXy>`DQ@EfIhnX2kvaE|_vGSirBTv5lQdYreM&KY!+&cM^w2*%HxWf-)xtdsPC z*=oV`#cLP<4EyMHmYt2Uo!C2NoSnhICR3;9o1YS_Xe?{O#-%Xu*#)1*EaDQw}g9PpduIu||^ z6&)-pYf!m;I13&64ubZWx9R55VN)$UjNTgE1rrWo5L;g^j_CC`YDikkg`jh7L{W^m zS`ZdE2NT9ia}}S$?^ytK`U>Bw_L~ytPCVf@(zWJ77lIL!-q|22$4e1z*E^o3Muzr@ zn4_~eVAYf;0@gwO_a>d*_xHu8%Zpp~;JjK-sK-yE_kOeZ9uPxEk(Q zIa5;|{^O^sq}FZ*Gq^tnJI4NJnJq2L;EDP90)QLY)9_HdsRfD!YVjAL*E)kP2y5k! zl)S*oYQ7~1NceIED~j26s6d{NLOsm5d0d;S)KcSVYSRU%8GN5_L*GBa0vU~o!gH(? zMORj9E3_T1+@_RApQzG@CJ$+lAFUwZv$9pnCzAEDKHV3Cv??W2F9{UN|40hg){WhF zIzLUOc5RqO1Opba$+dz5HU7^n+d|peHvkYa-Ews(wQamDdNO)3m5^q!{mQn5QRLcz z-ManhG~z)%kzdLh!MyL9MqQbGAnTHcf=uC74pvBv5%X_4Era^mA6_iT~lEH|K{pO>BOI>fIt&0_5yJdM_s7CfPDr%q{rxLB- zO#p6+7aILAEqn9_)NwHE;$$Tpm=-POA^W}QQnPM2l@Bm`E&hhXTglw zfu;XMV2sey9tx+|G^}FTI4RG;QkXhF`n7mqnp&C9#=Kd%4`LG>Y zH-uDoTl|@65_nzqsvKzFu;UQrXkX@*BE2#l!Wmh0_ z87e&&dduCGs%QXBM<)N>*x$|!H;JivjbAiwf6?Q;)Gnw`Es9l+Lbdd0Qp0d;ctIcy z&SaTkk^@I-4*A!l<(>$3my3}G_8o~+r5HV_>U6j5ZKbXSA#WMH8dv)BUJY~qurR9K zTlz+)$&PJS4pc*tDKYs|)bX-goP}EO&JaHl8NbL~gSU*d?RGbwV^hKh^R0KUJd6+dzki0nGtZ`v%X-;TfN1`T5F62!Hd*3S)j(SpV-8- zRdT7nOLv&wuxyg+XD`MGEK-cN;V06kiWvJHJSm zvoFJW(!CK=${;7~|DU`!Uw8TZ!JL>E)lf+T=QE%0KO zSxxh9_MV_$t~YF5TneB_o5mrpk&#OSSQ6r&h<3zjn1h?CWo#EC&LW1{!sZ}G@32tl z)dw1>%?c_T*#yBm(kNi93xKVUda6^8{PeKKrKAkhglv193fR6CJH7o_T?bv((u8qv z*8(%TB45Jy(E!vL)WvZ{Ax$4(ME~{<0xDvlcuhYrz-HTXPPNq742{m*pR&jzh4-kR z12|C!wT#MuYA5x4)<-WlFdoKKBE$W|yzo01qgj&iK)er}zoHoRQw(UG%`yDR+=7(| z1_%S0=AVJmCWKcis%qqV;Riku?!mps(tX*h?$d6dcV*re6B%qm zJ_{zXx8oJo%}h9S;47Ls-#@Na(ylY(Z!MtT-MRI87|DJdv7~eTB2qI&tNQ_Aj~I2` zxxZO~$us0WlZ-QI1CVy8)Q6@OR8>lym5ZkpG#rCj3>}KgELfFFll$goWvX?)tGTEL z9o;dhSdffeBKrolcsNvfD;_gxXgQ5)>$3GS zlmX_Dubcr$V$8e>H+{dxb6(PG566bLk^QPV530DuEKaU$*8qeasl=_WLQu9;1PN@n z`p4VJ1JsQBO~e99L!0M#rCc2Rdxi?Xh=%yG!81}WCG#Y)cU9oJk61~ztv^4PaQyY0 z8UFR05tkZR(wWvbQ;TV9c%&qj6D&Q;*4FE{OtY;{ofn!r>)x7}k<6<&sE+?~JAlkL z91SUSwp~8%a;08i;O*opz}xyRpAf=|z8T|t#Pfuf&KO8jq9_2gM84rMFXKB*$T-(H!?Dl3wLCEm^v+WK_ignNk=>F+!H4rsQD?13>I?8TT)e3bu!32 z@|9B%ZII}6+EEKYM>;tX;mKW6xD`G&xV1CPgABgM{yeu&t#86O%40e~bV7^03~_rF z?e=WTBk7Jx)3)wIScgfPAB@A%ER{>gS5{<}OT7q}NoDyTlVfR=_xYYpg?o>1TtheU zgBz=#35H0$evSysB%VM_1fS(-_4Fon>?eGI@jZfRnGC~(-i?5aqwrQGuu8MJf9Tk% zy9Wk!|2F#nYVR$hs_MJ8Z>5nEknWaF=@9AeZYh!85*rW@knUC_lx`5DySrN&_NKd2 zp0zp8>wK>39?x^X_lI|k_si=S8OUNwto5IB9>?#PF)^+6=er%>C{Ym5&^hIvqaeWf zy`}{I^6A{`vi$2!;62|XztH~kPjF(LwBoRjdHm2TDtgqvuYvPx#rn_BF-dFo?p;VQ z7DW&yzY&tm&jd`>CC*zt)D6OD_~e2hp$S0Hp;Y`_is&EM7lNw*G)jnQ@MgX!Rxw{v zX!+BWfB*h%1(!H^=7m3cCI^oid%v&U*tqTS54j@Q2-h-SX`ON_ z^FixaV|RVDA&qTmfvU)-ezBqyL`Ll2ZziVn@-G}96qY8axGEYVN%Sh6KC9dfC+s}G zTsDe}yApP92TT1#-EZ8q-zVf$B5GJ(@~EGivX}ITX0D6Cua85^nlpD61Vjt-Iqo#m zgkF-HK?&Rj>LT?(qqHRmB2V-|c+F=$ zu+CVQ4YM1V3m`$4TIi=OCV}RYC0blC3t*y@+g+K>HNrV4o-$6Vu=WBCzHb zTbp&;Z+5>fDkCH!->6Cx>Hs>WuJ9-I?3xoboPzoHM|xo-)P(Svk9Mc$4V{z9{sf1T zRA*8Cjr~r%r!)`4MRuEC<&ilOmvoxMKfv8$mwC1B`8d4@&-g~lyTIDE3ObAu#}I+? zQAgu~?IZ;HSIZ@S5?L4tL53^ zZGzD}44pxE?f9zx1Lk09-$T1L^*QQIot;kOf}EO{-Kc5Ka>|wDUU~g3lc6#U>Dt!; zF1%|Huoey`OwQS`=D^%ZT&!&ZmnS1V$~lXW@@8Qgl6bTk+V?vwc1A@{g&7zdqNZ^iumF~4B6_Z=a*D{*e~cu?F}ouzlZU0P~-V< zP}gP7H!O?KGtG%9W^0xKtK8W!O0V>7?DB2ld0fz1D^VeKEUZWsp=~YCYI0HpI4DH zL9W$F7bSDnZ*vBe9Cmp1j)oj{7y{D(LtqepDh!hGsdMx2bOOXJBO~JYsCCB6AX#n%Yi<#N zET&*us3Y-!Tbrpq2HQM77mH0H2a-?0JpXh#D@Z`_mf!G)Hzv&a63 z|MttTKM{(K9bB6+rX{P*Xprut&I!vUaHy~vID^9I2@)tJtN5~x{=E@NthBxyjz|d~ zr5AjMb+E2}u_)ZDnH$ZLmkbOQ+wv4_DCVTVatxfl*}Esn_0e?J<}KN{Rh98gwyrnT zxfx#@Kg!MHoRSmLw_l5_V@nJyK6+Ho%#IV9rsyYUqrZ=+-B z1nX|sK9eUGcg<tq%^Snwt3WU?%qTOj3q2SuX9i+1((10a%@QS`EIDfP`aWx zH1DR&d{*f8JBq&055n1B-RNm)6Qk$w7YC(*Lq*OMLs5m%*$!fFItnNDW}QRst)t3C zw1gIp2sZhb8&Vb>v)A%#<=1u{%hy1N_Mv4Ozg#_>5NJp`2F}G!%{C6-scs4kT*%Zy z@j5X%MdW77%_NQxp1v>#<1x7Jt*!j|DwJnEY|@`ggLMRv*qxF}APvLf-cZpcJw|a4 z5E6KRFkU*!aWgM#UE<@g2+5VV);6V+Pk5JG#TF0>)%*0fx@n{h)z|QUup>bO;FxZ# zjtGs~6ub*k9JocJ?y-oSm-W0qq)1gPKF6G@{RPLk2`YeY2-yN|XVhhRCU4ijkK|Z< zo~jm}9Z@!Ap5WR!zaK;tihb57^P2p%8vghsEnRxVJNidBrrCuL>ZoEW)Pk%~aU$Z* zd+1ieP?n#Tx>&V`NQoM}Oa@Pu_H~`Hkqdj#Le-r#HLcW_o^n{Q|BB}j5k0wvyI?ilbWSeU>s5GZBqw=4yDLc#@ypu5?V z(=q%z0i&z3^jlnC$Y`=e=`s-}-UtVtjmZnn6*EMO>W3zu6P1)U+4&G{zko;{XXgUd z#?P`%cOITQ&g{0vkGmvyKAVE3WMxg~NCrtA^=Xbaht!Hc%*~&m-$h!TcyI#Ekz z^fvg*f*gv7_)&%be$S4sD~N!hj9GA>!8IOC1whCbCZn zUA0(r*VPhytkj$9!Me+e)BqG?>AZYKC*lQ|b?~4h3!K*x4{{n9?s?kme9x`zHy!jb zL|Uom-zRbW_?{mjDj*b-_EC#V9XhV?98V~*+AUxy)@dx#-eqKKkV>hj*Ve5j zF47qW3Y|QC4yBSt)-E@nyowV%(wQ3Sy7Y=ub6p@6?gT0AVi;ySZD%_#)(<%rOm^Gx={{(07QP8`sIP z%6;)oqk{3rqWgK`n=jgc4&_T~ss+%gn1wnWE{Wja{AD*YEGwa`tYrhQnkGFCWjopE z<>LH(F35F&&70tjuh#9UlqgnM9=1a4isq6W8oe^ZYq{up;ibc4)1&$a`hTGv#EK436RLr|$N~wNqH|#VyUG~BW>l-u zi}nRZY!=DirN&SzcfNICOPcGwoUE+r;?4Q0(sKlV#GwCZM&4emRXnyVur6d&S`p`8 zEexC=#oM@l`V^GA=kW;tG&CzRib^;^K|$cFBeShf6P8YZ>-GthtTH1JSc}2$^!0r$ zcVu!FYP;l?KBexL2>zc4%@)e#9M3O;O_4$i>U>eRbP)Uw?XjH_STn(1qp#iME0JAM8^hUJO_WMX$b6O&?CD0?j*e1!nKQd*X z0xQ%e69F2I?>U~<=J2taRN!hJ(4zp~xyX82_{d;%$TyW+iRWmY3nahM=$av{p-O=1d^>(%L z1)n4blBAjMXQnbrx6pz#1ooxSH(O+g9*L`V8!~^=bmoS+6ERKITaeOF_hcFmo>tzr zsj>JtfM%U)5P}DqgpgjHB`k*sE3f>-fW8c#8L@N^-Xvh{zxd@yH`pMLSNj(4NQ5%o zy-Utm%XX6^3j)5s*G{iz%ox++yEh`y8`)Rn>Ngq^tZySL@pR%{XL3w(wRc`&!#cuJ z1IRjR%Ji}&=oF}?$(TWjOkgv$ZmP;Gg&dhpjL2dR;<71|QJQ-baueWe5_|j2DA8)P z@BwISOaBxW=WXN9^rz2!B>I+X@;ZapkP}N+X_~2_3}!K_L{zX0Pc82AH^4v>fdQfK|PXp^ivBu|M=b0S8i*dcxmy9<4d>yy-i?udm;zu;!ExzOCB*IRS z%`iT{pt9Vxq&bIbqYe3)ALGSIo=uTGK!>!S*HSmRiP2YIC7rnzbmuxth1a~P8c8L+Z#cqE zIrP1m30NKT%AxPVnAS&nm99_iM^e{o<(3MPv&$%}(2}7ziBOZ-t)zv>4EaD9`2t)Y z^h%j9S^;QAP_>%)ncmad21_?aSv^wuS##`7ug^}yO-a$2yz)6U2E<-paP0Q#PkN?kuV#%X7q`-D{$D8u7lef z7A%#l1)PFuaae3jFM}Srm(@M<@f&WgCX9hp3ID2~TecVy100oPk9E2e$yIM3m#2S8 zHk&Ch$%fd5PVVg2(x*KP^bE4MHa~ArCJlErsxnhkdvlq+Yajh&JaV+=r|^b=pDZ=< zK2CXZSoxxK3h_aW)aWoEac zE4?Q&;Aq+hB%3YBK{&3bVFkXm%9Pv%#l^dWig$UBO#Ug_D@7$8g0xr!5puxT5on)~q84qYW29HDXmOb>U}t zz^(N`YPx_gDxB1ML#f6pc+#>j1CY@j>FcTKQR>(@+(ms$e+37CfJi?IwF7S z@kB~bRwF(E%$51IclX@A<92*8p+&@R|Bjrcum5sku5rF9y)n}b^Skvsuz{nLa@d3sM?E>yoKHQ5*Q&sV<&Qh!<={y(egrp(FFre7Z zl)geahCAfx-8{QXXDNS+9cUt^IeejHDwZjVOBj|BR|Dku3z4hBa{P-FNJ1*3xI~A% zo_0X@72nq~=(@#i8xLjLnnwhmC@gmW3=$-s%y2>{0V;DX@33LvS%rCk$oEqaVBRU# zyaw8=;(Dl%j38{H^!@32xR4Zx`djH4$ZeQn$z zBjJ7l?JU13u#`6N2+a&GO^Dm@$C7T32oT#8h2LivXhm%ym)RCnoWQq z_xk|NgNf#KYCXIDdmN3rwH1au^05o|heU6HWeSN|kMn7atE$R|UKEmgI|I@F-5?jV z1fNjG8t?uo&-YlCBP7WxChR}@OJ|JXcG&r@_*E{-*7`ge2zCqXTF%gS^WP`f%A0Xb zE5>MGR-eLP{E?Oj7x{D;j304nBZeWEhXRYCgFt8jJK0my>uJ8LrnSr82D=bnx8}eT zipzsV8Y=OkP1{+**7M)bxWae+2{5htV6GhTy|;0+j9b=+&N(?bf!66B^nFnyGc<=Z$`*j}eSRuhqvci}12BGN zMbf%$wdwL6j-c1)cMYD206=o$bH59N@vk@k1>?gGNUW86kL40Pl(%+ql)5iN_ys|{4T{Cg)whlcsn1UEC5EbG@Q0K+50IgV(o*xAtjX*%rxr{c(eevVvvR0#K{9!pRpH8CYM&vnq zz@)W+rR-}rNdlr7wF~QW7Dee)UASdQR$W4rTM3SU&Pm#~ZI|)sY2FUpz{8j9WKjV7 z!W}?TlM_U)0dyo*L7P%Ycrz-%8Dvw+ExX=k)K->~;atx+VjpUFSxLQ#alrCf@C#PN z*>&%h=hMHV!)P90(P7|}ou51&)E8(ObBVq`e&oPv{9MCjYd4Iux`~=f^(>Y&)-OyA zd9i@a$wuS?o9ys^(vAoVEW-`H0>0c|RaM7p;p`Q#Ko&) z%pWlE?&>|_DBE8kS|UkiH%1us{*lrgIVQdzJfgnvyp8aAAPm0npQX!n!)XLFJp1AP z0-_aE`IX|@-PlrSL{son!CCVTMhfn1GM|fp_ih1FzUqLAne62bfccCq*dD2^ z<|O@Um}MCQ2bd%5`U(3sfNjt{k;$nG<|z|C42y3poE|ol?mND@EfIFr5X~xbxM|?6 zNT2^|kJlKNf4*YZt+m9m`+Kd`h?Q6r^u>~rMxX6j462Bx9!S`XL^Nrv}YdMXpFrHLYC*WFf1EuTc~Yu}QW zrxa_snF#RQcv+(*_2Ub)s54y1|0^;KlyW{Rr1d99Sy=G+F)|FBp{FR$X$vrvar~1Q z7SP1=XeisK4?c$y&3!SfrB~CiHrv$?%qt*|7bn4lY#x5>WFBg@bMu$zJbyyd?;oNvW>_AE8` z!$ahmL&xh+w_hy09bZ>qd!|T$83kjs5+&$b8aAy6Uvi^#7ZOe~ei(PKULB#FMQaG6)}{ZKGM66b0g zDcOZ_y<0B9^l{|JV_I0c$?Fs#Eo>Y}3tNcWZ<4)>Xjh2Kdczv-m+E852ebd-7<{7( z#%U=ZW33|*OMFK1N^lyu9{mx>C&hb(Q|RfeIVn4h83`sEdi0XXRGN7DZEgM&7MA1- z3k!=a6FpnV9IH~n2LzX(8}9ZARVMpoANd=>)~1uQ?U0FQQGk~WOcX=B#F|KahZHpT z2;_$ty*;RnP5JSI3W#!(m6JQZB)p0UxMk}7HSNse9ud31x42AweUVRU`oIvjG(tVipa^6bW{*%olW0~CHWbb$hu-cG zA1DiSAAfeQ8%P-jtzgR0>W;S_V3=zH)|mFWp6qjL3r~_HfnWq%j7uX*zG~WZw$p9} zNND@*-4+~2%mk@r)5t5e-QOWR+ChW9QdR=e?H_5QtUMxp5LF zT$lY~6DQlffm}Q3%_2Z@OP7|PmbPb9G*R>Feh>nc_1GCQN&+oPJnzlvLGPx;g32mna3vv3jH#+Oo-~oFPy&ZYaueVBny4l3@UZL z3MqQmrS&y2!1Uf9v3vjg?8`yGt8m4x*|BpCr0Q=wcGFD&X)&+sWZuW)Lui%SK2==4 z8S{MWfGf&MQ)a9Rm30kFP4hW5`fnV6LN9Kj$BAg6CFxD(k0lCZZ#Fub;%aD{j~kr9 z*Ce&FLu+90uDK!tck(2xA|{uf@z_F=`*CU5WljU5uNU0>wTNz9;$dB-M~y8|2=7Do zN-f!+$noDD5W=>GsjMc3ZqWzSudnPSEE*n>ev>x>Kl^_ER+vskO90p!wrN6jh_(Go z+pQzTCLE5XOB;V=3D_DI{Lu!y96a*Yp5K2gv2&Gu*tyyLRl~Aumzr~8=ghKc7xdY0 z9Fx3vyC*9sfQL++cx1`y0oeNR=Dj`OA!}y^hFl?iLa3A~&!Yw^iX-T&6=Bk4`g@j{Hw5sLN8P58S5bd5tNnfJLhndQfnOSqtc`LB~sDnR=; z(Uc-J(@ls4O|FkL|7W=v_*v1-i!_h&HX#p3sdSdXnbI(b2Xs!6OESS`jcblt4~ z_3xZ8rb-|ujOS@t^I`I(cqot)W-0!d69&y2o$Fg-ei~H2l=c~z8x&Kj06AeN8XQ0m zDcF+I-V)8|w&r;fCM`e?ATCzFT`aHa<#0p-uw%Bdz?I5fgO1PhPZ%` z8hE4QaQ}U;STTd+K4nQ7+s<_EZ9mLGmj34E9bKB;bi52jN#_6z#W&^!3fO0$OtAU{ zaO%&_oC*;W{~=6oYh=NT-2U`bkQb@7b@?5A1_3;Bpxz^kPe~urm-cxS7lEC5)~7g1 zi^1Lbr2YirhbKqMPDoqg^5n_xH)mb33*%3JqCn?Tnd47o^&)On;~a%tCa6K8J|X8r zwC$E2dia-0CgYVl&R_J^n%l_YN3Jq>rkvd0pM&6?r`-;6r$*sfDoU(Saph$nz|VU$ z4${Jo%^?JIeRHJjMGQ)F@2k`M+daD)-dClsRTyM57TE?`7X9>uDgrasLd1#5{(0ar z;?Bfu(&p^V9Jk9TBo<`p&Pwk%YIDd!xJ_&dA6o55sCl+ul!Z|Yie5db5BT$}QryQ6 z7BxC3V8NfV{k2~uDw$Wy=^9k}5rIB5FGnZgk#ND-(4UdE>1Sa#j%2p?M_}Ywdh@t& z49Z}lD#aK;GYXz+yOv}B7IyjWg3-Ihq5s1aQo%x=caK!XY{D&FE~u;RZxEmP9}pi* zXKO1ru&*U>vj&8OML@UkUjhw`!)-#*`6PLNIv-m`v+>)JI2_*5wrx&_;w$ z6~EvjwCzzj+HVHhM4(tbyYAInNT6ayNp|)U>A2(Ca_G$}8+m!V2F!N_NN2b@Bk$gR z9mZF*UcX0w57fy$Gj?swb962jC}VEtOh8TzCcx7YXo!KMVkSke)H#tSw;IhQIkDMCTs}3{8s`_71E* z3Lkwh6+=ZuEjDNoZAA5dUEcbi3df+aA^tz6t>6FAZb z|Mw~hfVu!GN%}kvkZ3IR|0I*xoJj5wh*mN`yaAB>;eLI^y5^BR;;j#Pj@CKSdV&TV zT5c*dT38q3o_jQcOWYi6Mt^pg3y0Izv%;Qrs7%#;0E}?Nu5w7kSVX;i85hBn_+U^o z;ul*$6#75vBtTr)f6+*Xi#5*~wxR!Le%MBMuMrGdvz?nh~z#WQz%3y~V` zzL)N=cDIn+f9gKH3+wgNs4S$hkgP#dnW4)EM9n~@wh@#KWu2v1lf3! z*4|mfSjJDJQFzy<{1s&^b>Qfxl_FmBhenyD?xv_BJ=egH*~=IPy7<6Amo8z`R{-eJ z40?BVIgHPe$O`hwQQk|`_Y8lVI!crb;o1G^UU;XF26kDqNfeW{rcsX4p<5mG;I1_L zhjRgX)D^-~wr{_cFn0%FJie1s=3hKM;0UA-;3e%HS-uX@N%0+tiU)#>YaEtn*~P>l z2v1GtDl8$;%@36Zw$?ycm@47BJ6w_52QslTy}#UZ9kQH~-rh~iJ*!T#@$vD3Kjw2D z7XFZIF9!KNx#aA-S^8u_qUuW#M=EXxx7bZLk6Z|br%_);ril|muL~}o8n2mV3wUm% z&Q6d`l_j~5oHMUsc9Zj`T_wk7Xb|I0SZTar-j~K6$T7B$6Q)vd0<2W`niUvDEqu&3 zY7b@5yfD7Km}jmN0FH0rrNTb6otbAVatY`ys{5tSHft%p0U1ViC(g4V$zlE`$E#We zrkeD@lEMIqCY&i9=sdi38d9U#(*=wDQq^W|UA{6K1 zI5dppWX3F5^D159k#P}e`NRV-F4+L%QcE}Q6?;V#m`p1Z_z#OOltNd{tAI)8W0qwA z;`3p`<9+W^3|vn*&2J{mCIILpn&)fz`LV@h#K(duulfsrWJFoDoWYHa{DZi#KEMzc z8WZoWI;kw%43>j#6+&G!jsuAjW1+Y{J%c%(8T-A7!MuNn=QUR`0JSK9h+ zCF%gq#sy~@*t7uvkYeNF)($z3h>$O}Vr)v*byxP9*QtVif4*Q`GpYlh7^i&CHiuL2 z5l)YySjhpuaV%e2`LoKO4ss0Pk#OP#GV}^Tvg&_wf7%hC=T5Cu*i&`Hfuj!k&NSMYIsbPLJp+M!YXP@*yw26`Mj zulmE*7EjQ|Mhsy&72N8AcrO$L)r5rR8nl76#W!F327-SrR!t__w;6v=YWy}`m@_-P zOZNBX;+OIaB=6`#5-U%=yk{)eU5(0jusYMhE|f zU1E2uAz{c0encaQ;=Ri(0|1Dh0xoPb3wyXEI1TJ7Ch^_fika=B`b;cUkzJ5#fdscy*Lx`pD{>y5mcVP<_ywt2KBIE z9%R&g@Z7k|u@gLbq9Xds*hsSP#}$Y9pU*H7pRYFlR3#nski=7YV&v*b)m9OYfCd;5 zwmBD(ZSlW`{mk0UL$~9bnk)7S=kBTgxzZhHbH_2rA6TSkHbO-;x@bu{lpmF8i>u#x zEGdm5K#}7atv(ZEkylBZ7rn|=>&qG_DbXZ`d1ELq4J3vI-0vY&!xF=mM-0nx3tGJp zAWvUdQYk5^ul0pkQJ2juV;m~DYy7!Ds6O>7r)6ILV14Q^0~E!-`{s-WjGx3gT`h8s zj(rtKZ)e6GQ-Xo`bBqmjD+0WU^+zvW?{V}oD zq&Gp;b}+BUkD9SEx_&z$WHEu@gf5!GL0{U4E}i zR-}Be262VrEewWm=o4c898&`7 zu@K&RPcOTCT%~}qwn|FHl$}Uo;CYz))O(RG%Nkh??U`o-+UDo|UjboO6D_wOu?BdI z9BOJe`>)?<)6QtP`JfLR=qoOLpICxL8Mh+g*e`SBZ_Vq-kmy3mJD{$}_jpzTcYkd| z{?O~hGr^shLCS7lxN-tDyhc&PQBf#+DP}j;FF-N~>}nrR&@*l%*7=Qqs&3jC_z2DlJm>=Iv2n~ zYnnGv3OC2{)6;QD^F^SuuBn-?L<3{2syB4G3m1$!y9FW%B98x#;4iGK3(vKLF>LM| z7w1*nk8>Ai6{?jo2KMQ4iz^Gv4&yeJP6w5a?TTE#=+wY^74`R#>0{Wde|9 zjPMC+u)J~^wruNI68|uFjo;qXAK2gD*~ED{C1gw4bv9I&jdn<>+FJ%wtp-FzW^J zms*C9Tz7E+6Z_%rTcSyzW{xVG^Y^Y)*xJL+1hfu-@$S1lFYE9^*xAge1VQB&Wxn6Q z4ok65mz&--+LR$b<@`;5P|~BGcC4FD4zmI;f^>8Tm(N;zyF{`qQ5lNviu7qLHXrz;`?yuJ6MAbb8>yL zPWNPrI&5hXTYd}cVod!gqexKyky*q#ooW8Jn=AM~9AW=c1!hYTN07g6{dpujciQ3` z758dLIQ0Vn)$o^9eC`{S8cyZ!==yPuPfJQJelOi+m|PjIVpW3iS-v4$MLBh)GZZ)V zOQjnj^S&(1WcE|>g%E0`qpa_-6afv*r4@UC-dky?GE5+*u~SL=3ZZ6D>|m$M@J7Cf z&EUGl5i!1*V$kjvz;*Ek`8DWCk@7ozf-e(Qx;0_LDeM#Vxk>v&Yrh&@2Q!$3csrG@ zKSO{M->aaECb{3UwlS=qlyY}}_!y3Z9bLE({p@GvvM%H18BzNOcOoLae)Ly#p_XQy zI5!y5TavK?}n@L6|r|OO>cQ!M!x)H1i@GqLWQTqXNnqbghT-QUyP)jU+E*VcCZph4l0R?;qTdc&=bTw9gC=X}vLI=4I+mKiLxjMVJwSF<9 z`EQqzubfBr%J>)lJ}$*ouAAs$U`VC46jAc>qIfb;5jLAF*2%!YHi~?vmG)iaUJJ9a zIkr?q+3ANP6E^VnShDn(=!wSJxJCrOn83KldF!s7=2|?ax3>zj1DaY3F=%J2YBwI& z#U6dsIksY+%O#dOd2r9ms4U^sNI9gDEbz@(TM`}Jde40eJ#BoI z-P~_)d@d$Z^KTGN{UTkos!q>}kWs1%ULD?Gu$VLqv^;?4!?fp7$9|Gx};eS|u3ToEQe4xhMZh3g&k@)bG z2r+zi{)`5pGxj>_Wc-#CiC?hQ*l__cwaGv~uAq|2WVE|lq}C@VT# zaVfRSo357M-DH|Go$s35sXPliXFwV2D@kriak7lX5Yh<(ovnT9m(_~MgGa`6`u;M! zfnZVw_h3x4UQM7hcDU(6!R2S{3|6DO;6N{v;CDqN)Ae0>Vq{~wrjddVjpEEAWu$Ce zVF^6Oh?|;lm$l!{h>_nAa-9|}zZN(@uXi>V^fm$&Z11cca5+^CSdzVW7gyh^)QdJ2 zde$ZKJMAcCob84!b%nh#cS8B&bz@MtXqE-vMqE{KG#c=GBO^%Ck@9OW&xeL3Ls>ZE zPt{GKn#lwMb(+h^k)=duSs9Jks9g1pd&bvq9cf6#Qo=Or2~+5EqpmF;E;Z>sl1`UV zPs%@uOBpN|AJs4)A|WvtpwIXx3(jWpU>{n+vv6A`g`{m;P~5X{AVf6 zn)G~RAIH|w_{>2h&@y^by`bODiwX=vN1)c2;D-0>jC2`$XI>G?bLa!T{aV6&4zy_v z;IqBZQk4X>OagRtlC-lR;RPu9zE(fo1e*84H=H-F7PJVbxxhC+@+>8;v!a1K=u>HF zgN23lr=m2fpm*-r+IkdCZV(LjSZ)o1P%$-h-%pknO-bx!CyooU_4>(vz}8`Av!__O=CV!rdGVU6cG2jB4PFWAzD7 zF5&J?J2Va51>Ky9tQE3|r`mI^(*^c5iP))O!y*5Vp)dF1$gxWuA5C>^| z#_X6x=j2j_te9B5lfgUl#i%#2Z#TBR_6>JVtk%(;No80ygi5_s%rSqgTu+s$U8~FL z=l&+!+BvzF%6xcvqwiwQ_T^M(g^k16e$@6%Vbb;G1D5D_hb`0-<0nWD;X4UZ7K(Mj z@%3#Qq1wW2^ga;qsB#c!aO)lEM`2PYJ2}{q7W=$QxgE`pwYJM_stfvcV@k&tTdH`o zckh~FZw7*o(j_U}lfX_mwwpM`MQP^^ zHq28Bp3gsN3F{W9PITgLe2<>tef)Aw@vcZX@XovJq%V|7`PTs3NmqAdqcLPvaSB!0 z@QJRp_EZ(5aWsLUs_ zKlzQ1+f4$+_I6>Gtn>}mUXiD_k_k8VZnin$M2{GgNqVa@5BU`)$Ik*^IE`(7N|SfP zNZ?gQyU6wIM9kOyxVkhx7AGoww_Q&J3I}>DbHv^X2u2mi4TqZuhn74bm*UZ?N(&Pq z24s^Fb$P#Jd|3)}w!ATqqm@f>q{8Q#Q8`!(FDYb50*bJ<2<$7#EEyuGoKy~T`R4}? zA=n+qqKXf3&O+>HwNAx2ZU4uMe^d)piYe&hU?&2h3nNJ4rG(gpChP|XtM0(5&wC>O oDdqs9KEwZvDE^Nr_WA>|$bBkX(D@(PC%_*$DP_shH>Upo3!Z#UQ2+n{ literal 0 HcmV?d00001 diff --git a/docs/topics/_images/network_01.png b/docs/topics/_images/network_01.png new file mode 100644 index 0000000000000000000000000000000000000000..1788ea76a884912208ea2d41cade5866e0b4ab9d GIT binary patch literal 10720 zcmeHtcTkhxzh=HR6cH3eP$?FA6G1?_iWneLL+C|7KmwtMUPJ{{q)3-4og{P;=>e4@ zAcT^D5Fkh=n9zI44gT)_v9mKfclPeg-rdK3Csk2wkpZ*6Q z>i6j{q;BH<5bpiL%@cqq`>8>Pn7rxGV|y<7A1GXF(A?QM>$-tcyU*#l&@S?J>doI8F3c60FdwehqESh~}WtDpbdxs#Wj ziw(W1C!nSU9ZQGL9>c~qo~}-=j)0cybCGn6;a~0Ho_6+tP1@_F|5i=S4FkU*f3BSbGV7-zjt$FN+!C7 z&vn4x)S1WW=V#aM%UTw(&Ry)>&a=l$1Ymk|>bN-b=)`ex^5#GE@0Ve5$A4NL z9B&5T*$uwqLh`?FR)#pNRWY?Zthx;DARc<&`2lgls%5cm<7H2#>Yu?yX*7?~8e(LI;>;83orqpn^7z zBH#xZW;DgsC3j?yQ%>XHK<=mCxk?hR=i4Ua_VC_P5~6M0W3G`0=Pi*4u{hfMF=5y( z0HD^}{x07J!YfoG_Ndl647)*^^^rGh%GFy+nR6^Ml{Rs-({tVu4B?g^-NUf%78R&V ze{X|Z*fMgfpn%p3;O_gq=QTY8GtBniognPoY4SP=06Z=IV^Ll64LU|Bm5H7{e>VR4 zW*da$r{nCC2>~LzQ7xwty4=x@-o*837W%_*+tFRzy!U#H1jbwE8NEHgsVhk1Y`9@f z9+0T>lzK*e{v60U;NZiF945^9REMv_I5KG8PB`*PK{uIfJQ!NQS;GJT#QpQ>!nvF{ zGS4zZO_!WfYss8+MZpkW06uyY*_elJAyWqni|PrQp=UYj`%EcgDGE3I*0_--iFssE zFEkRLGte&}mNEmSNIBd)33&OZ^BFwPN=A7Q74i14x?tV1emmk(dS}XA(~dy&K0_07 zSt}%){R5fms5T030A~G^xsqy-*`qHr^2V}exS%v!JRaxlCGsSI;BzB)Ts#!Ha@l!*>#h@l2aiTJ?JqL-rMKP6AREJ+F8Z-7ohfKRr%&6#6BQ}?1$oTcA zdM)81xyhtoOwC>ja2kgLk>QkP7Y@9;HWILtH4;D_tV8}x8CxGing+c~rqRzNSU}pB z_0~;F@L-5g8xo>k@9Sh`rY($0E2S8v3{i9?&W+%?j5dMeYy2CruV6PZnL2(RRxt;> z5`D!drEb|^j*zO0UX9D^#Y*G#X*m@q#FNuvwovxZuUjzhc6%E7!?O;e)}Gv1y!J=G z+wF|RT?XBerW<7~M{)f++ktBXLXlS!;7TQhL&au_wu~RRP8codseb}Nj)Hat42+~% zjh2?^9d4p9#GtP239en$I25CGPGiE=O9`0i0l_7c^0bgoK^97FIuJ2;#2U+*p^99> z?2~=`CA7a^8R!V@8e2EB$?gpDH{o8Mi?06Toc#)RNrr~h73aPS|B$+@ly!&^RZI77 zEVaT=VvBD|t~3$&2PDa|CQmeyrR!*XgY3D;-ULZ)hbI#{BCZFGs8#a65D~@BP5jnFYE; zkRfVQ-zO}TeT_YmhXOdQ&rMZs4VN)dcBi?)D-UX9<+f0;R@#~P-hxSa8SDTm69_I( zv2|B4^oL=oO<27w{g6jKuPhp+qrfBz_!D0N}8_-IvnTI8WpMe}$-g|JdD zOB7z2?(tvrtrB^+HD-xT0w9Tk`Pb&%st*WQG&bFN;lSR}_UrI0G!QrU#`q9;!? zqb<$02Y+L=-(KH%qFw z^pH8amDUuqm72|K^nF@6V57A)2DJ3GVn~@S?fHJDa70flW_#CF`In#0*QeTb=fVxQ z{Y(^fb!K`zWR{ou$DUG?pIc&V?Xf_s%Bjz*3(vdsgk*zFCbQw93C z{*o+50N^KA0uSgC`7L5_ZncYLCR#=CnPDAr=OtJnc_-yw;Sj0S5Kh7Cjxlln85)=M z;*XFo7zm2Qa~L}!In?(sy9ifGeYHt}TEird&^b}fs;MgT+ui01k}(Bxq>n3%pi*1} z#L+9gjpA9nm*;PWX_b*I;U6e}fGQYgh%Y}lqia&XAI_t^z4*v$$?XYR+JmSQi7f9P zCOfas_V{X57C-6`m!U-#>^*qOtw!`wgwuF+Cu0nfW4l*6YPr_x`9)ccCDl8`e@&oz zqUH0L?5}~aA{^n6&|h;16N{ex+fbE@`Xh~8kbTeDW&vIMtaBM@m0u-Oa=)eW)r~Xz zTN(((@4XLs7HR@t_cUyH{y}=DqyWY+V6uIal$Yi#Ci8)l)zFAMK-Qf>Of|XsN6eix zNme*vLpd5&kr6y)Aj%0D*A3WQfiGSdc2c^i9Uzt&Y@BbMo-L13zV0x>bh4XzED%qn z7UTLnx~S!)YP0qU>BaLB-THO5P>^3E#kVP5VAM4IvxY{4^->KlIdK`kmvkDr8k_2M z@f>Qlt%C9^Sd^2Hr)>kXzFW%@<>-u?oY85hBpAd{Xs|mmiV>pEZgiNSteHuBb|bfmw{w`Rt|lx{_`uu~*wn)5gUUy(DE+S)Y}d+#4B8$;haFaHHR64-ply-`A8m#6pdYk)hWQP_T{{dpAL|R` z>lC)OFADTVtwFD9a9Gj+aG0FNjsan-)^(Gt=8XtpR;ML! zgyd_nhf3@<=Jp{qUlLi$YT_^Yn1%4@cfkEcKq2$RF56AwB~5osp$j@8O`9_l6(i# z+4w~3&+pAa8)MP(UlI&XGL+ym+dR&rev|eSBES1tjPA9~#B@vghC#7$MdbWlya(k_ z6ZclH&)NJ_QOWZ}(fVGcr1G>pW#0hw(deSthG?6|M?-4&a2dQBc?d4@l9_2=% z)tEQ=CKe6pIPFIH^{CC23tP<$ibmEJ;=tU|0~ZZBTJ~pIWiO+!Ng*~Sy6IMV#_^U5 ziS5sGBatyol1hKwliS zryB3&Z$fxQ4M$8!VI6rGUQozKxEONo?p(;d=A`!f!EO^9;xALCYK!@d&5vq@;0+o6 z$Q>6f)aAW?c#H{@$6g3$q6E@HFAZ5hVtf-d#ma_!K)#_%AFvy*4qJ!&aDOhns z=JPM>0{5`najO&sg(f;H@)g#JOS6Y6uxt~5p7TILodUi$De-zDZY}nwnoV3|yer`! z6;I-DSth_p_e;n31MpY79))LVR;fz7MzP24V%U-Gt`i`!!XeFx6KfWxCF0wDXjGR> zj*ST{@>V_NU8`oJKIRM2nA3b8=)@Nvw0%mgNliwM9c6oq-<3UqFplc0^n#A?8h~z; z=oy!nOM{%r3u1;n9T~D5d{a;eC!WMQU+vGXg9Im-9H6mM83)Z>52%L}rI!KX<+?_gwpx}y`Q5m{P1 zssN*U)=ohKupn_K*lkzhC)eHO;(QCuI^@dy4=*ME;&Ar&PHte6!1QmP4cu#YxJaCP ze>6QTW)f9BWU!YTqr7H_93V4b9wn3M=R1!%go}R`j}=b>Tq<{nR)JqJYBp2TDlegS z7E?o1@|9}>&M4OC!}%nHgf!da*L2a}Ow2Nn8tZ<{oT=OWJM{%kK(O{@mQIRKv1g@M z{A|o%bNV8-y`aljT!vqXXJ=e*C-eQWXZFc{;AYMrsiOSW<}D8l{BeP1iPW_KBG!xA zy`WWVcRs6<7ixHv<*f+Z6%my$%IDEF7I^e1`<2i4nf)oKf>d4I^_(BO%lM{zJ@eQ| zZ(nCJStnUWG6FlRVdxn#W1fn_(5fYE#%PB0$!dAF^IVg9KGBkq+8TH9ep09WU7fo5 zE6h*bE%}9Q5qV9G!@*8PPxt z;_!nOM?Ge9$W$crD%(?sFF`e3L9V4+ny6zxnZ=p7gt#Ich1zvDkvU&fJ23pX^&8Z$ zO3;3`;A4RcP*go#`-?AkZyP)Q`D-NrfTOU}dM+~?rKNk%UmZWKFO#MBKK`RW68TOf zPH(PaJ-UKCcOMdH`gqeP>S0o?;oT9W=2cd$z$QfM;Q>>PUBuI$Iw{~Rl*8f#DnIbG zGSo@a?d{Wfeb7?YQ#WtMlOYt?8o>P%*lajoW z{{S-l80}@f7MGVrH!X)vBG%Vg=Vvg+)Nvp=S< z&ZkVRz6iEAyr9ZMS*r)x^EY*>jZQ=(#08?AM;{czGY2OgOvNcjBHdQ@5dWo`U06UA}FPIG~|vRwsV@l-F&g^yp`EQMh%K5s|LuM#*)g=x%idp z>cyluU%g}_755&s$+P&)ifhpHcE+)4`Bc^K)V`8}W41M4+luSfnb_RHfbN~9- za6!+V49KL(oSnsdgDe|uJf`yG+PfLTP>1Nsdc`#Zt_GV9wNWQxzb|s?^}O0Q!^~q} z{qofYeX6NZzW8}cwzD$l5R9Z$x;Ny$#;e(F(6<=pmT2?Ogt|Y{(jj!Vazm;CRYLbq z5QnzbIk(3H+}-ZYzVCo7^H~M|1QR?{U`2%p;e41|eyx;CIAP_nNdxTK@2nIQGJMMo zbOpSQ6Q3pxt@6N`gIzQ~8d=+~uXf}iL4+^Awdu9L|7byCV>1R|vDrftg>ee?tjszK zC1fo7K$_(^6ci-!4$RXKMOpM_>ae%%P_Ino37$fyku)oLfeo0&5f!411UqkM_OFYT zdCH@~e-2bj%R*rLMPW&ayunV1y>$fCgXya>$~o8VxBH6n(OXf(^J!k@st zD37E*YF5hYDc?rT#uYU9W)n7JPhl1t9PykL^2UNKip04Cr4hLR_sVZb+QWMY&R2Y= z6$Z1I=&>D=I9|{c2|iVo=g4qm{cJrk@L`zT}!Msi}8ip6Kk>}&o7B!hKd9Qo_C@K#599wsCuXt_H{krOgGtWRLDC09^R-bWur z@qamA8E&Oos=A8^?)a)WjOc2tHJgY=4aOtg((*?6UlY$}67ooB7vr(bJDtOaqEV~@ zk?ofi0(2B|{^iFTpqYuwlLtObfR8F5%Mh5aj8(#j7lWO)4 z8LjCtRKOJwKo)CFs=%TC;uEy1gx)Pf+8y^76biptlhE((Uh=WW_A`DY?uKTL*XVBM zTQUV1{xr^4Kg%b0=r&P4yONHYJl>q<5&o@*At92D(vu81W)}%Wwz9xT1LIFuSx5tq zVAe^wF0(>mf*(7KKl29d1p$TW(f*PZxrTROI#H`h2ja(VU~LgbYR5b5-(-f65+!rc zhaZNN?LzIxDLy}FRz(rXk1FczpkFc@|MczQ0vVk5bRT?)0o7+P6UxN(H&aDji1+=C z(z7dE1lx5odbasA-y^CkK}%b)dG8ari_|dI8`qLL1-Mu*8-0)AA@FPO5gy#gPPrXg zneTNMb0AUHdnwAdfk9N3p91=}>qL@`4C41V{YS-@X+m`UiI|~d*XZA5OTunFSJtgk zk_lZgNE=T{(_>*cZj^;LzFLDY{6VHt`jeYe7?jo$5??AF3&dEvqeL>H!U&Z?>(7zk z)2B4eGEn4uEM^Nea|Badylrrx`uoFonwf%j(E9`B5nFiflrB{<4QlQEOc9XB$riec z;BX?r{_aC34+Zqm*3{*KD8bwZ1i89Dkxp&&@bN1NxBBL#-k*UX+zc}4TlNy3 ztg0#YUQ)J3&X}lb$9pkf>~F?GazuDyp6;7?a&Qy!WR0+t;m$ z#H4tw@;iF4`Wi|medA2-xy#Tzhl{;##{Wz&*k2aYqa1kn?9g{<%p{+ZEy5LSfvwg+ z8OJEUGmpoQXS*lig_o=31E}k+kQgt+*oHpDsJZf}XQuW`)B7NEs_?)NfA6_cuSsjE zCz4vgGcny|=3X74bT4pYg`3|jc$QDmn?D?BwM1wh`&cxp&b+T^yNK?X?IW8jjn>R{VkeVzB1Bf)tYOVo(h>c;02V%)3+dogNjNYeIDM zMM?F*5_^4%Dzkvez0v@HL+Vc!Ijw=BXXYl+s_&3@ZSH@)SVO!?dz1ndIX}~ETao*g zQc)1b974g!O552rL4yl`L^C@wx-O04BR|vJMb8l`&5@fs=!$7x%!>&{IN zELSJQ*}`7t93t!6O9_xHkCOXb$EiV{AdK}t4R*7h#cSW|kfzDDIxqR!a>+`4dG&Kx zeGbKSzYJvA4{%S_w+)3dqw>)$pdqb0L?)ZdULy)Dl0HAMr^}AWy$9y*eae+$1#i66 z-<#YY8|CX7V|1t6c=LHcabBeu9KDI$oezfU$n)99fkigwiXHya8Z9CAlZH?JxOEG$ zg)2{Rm0GIo9q9zs_Wy`}gWk_BOW!beq!6nhpdeTS%|RuX)+!uT2GjFS4?lBtKGC<6Vqv_@#`E!I{|7LsD}_Z|0CE z`Y}Pv@J?`sVpsa!bNGd5e_qFQ`qTP-{HhOa#a(EdE2Rr$x#lM@_>h`FV22PVeFPh| zyA2k6``0_v;(Z^MX@NWi>jK=fOUhK>TokZ-?d4{*f@K`Usjzu4AJ^03@y>*?DLu%d ztI^+6cT*$I#4v}NdU?^9SeYs&Z@3qd(x@}f*-Z3E)!e^ciQhE`aThThEeiKQc4za@ zb_+WiRf?*E%#!p(+45tC*vfp1qOPuSkxab5f{^oZAU|kFRyzTBuJ?rMNv9f4rqt5E z|0CR0-0uR&KhL0b>)ayRW`}+ms9ZW5b8_ilRF>q}S$Z%35t{3NksbSgsoCT$HoATX z*B^b-+fpSzODPL229=J`@`Y#9sKW7y`VZ%1-oU{cmP%PN=)B@d{_9+R*1;l?Wwl3O7)&aKc+h6Xgk z5^1GkQs%9h1qJP4ocw0zDz1_97#F5eeI>7HeWo*IlT-K!)DkaQJHYRm_?r_DH!-~t z=_2%E4)`_hzF`AZ3c|u7BciwYL{l*!qGsAWt!T6+kZ_NklKteEmd&OJ0+ILIJHV5aZrWUicJFJ^H3 z4ccAs?>zyV_?tQApFg? zw|{w_+4o;}4+LZcZ3NcGSKnAQ*wvdK_hsQ%D8_fn+1ZP+ zX)O@gxG?@5Nux#ugjhGcvgxdzylu9*YravGoqA`x&v`EpzctN)neH5BzYi87y*Eey zE*r?lomZ-RXnh1kOx@ufyqzt^bFghY-ms0^%u%~T4kVF^=IoT$f>soQXTxyFwY41I zCxq3!fJ0Kzmhej1jAf&RbF)PrZPko2xi3eKAazqK=|To5987%krk9t}mmPcyOXL%C#x-TqX+@IzQ_5b}@!N{VhN?LVzT{k-}5%oUs$Sij3zO8x3 zfYVPp^+^mOC~aMUt#EWfONX<^x6>3t2XdGpKjg-Y(kV2_sE%U`n)apm@BN6jJbsBtdloILEETjD6Aj`eLvIqy(jV;iS`5kh1=xUV$2dNmCWrkSGRXNRq&1Ycd22CV4|Kzfo zkx)`~v|+N+)$iKA;6@`$U9IM7uMDPlI{7GNQ}G=2RTyfE%vF0`^3P9AIP;OCRm|m` z?G(aI#2(J|BTYGc?Wcs%h^;geliQLp_xI%Ca;#+((8l#SLTPh8p`0c(a3AtdXxc3_b zjpN(YtYkvV%x=+z;boL{d&G4DsruojW-N0e)A&k?8*+9j62CIM{V9|3s;%Dl@B!pz zcJm`@sH=WAN%?b9sNBxMtJvbT*Mj(ip_@%t&!YAc*uN8@yY1FXg>Fo+7uc>V8r&Vs z&gN+IeHg z+Def0=j*&G<_T$cDBF3OH}o-&W|w?(>9P{qfW0|IaRtj_^y0hSHhFd5Ro_2sCTnIU zYz?RKd6@~L7A|5Q62-H>2Ybzy@pnD4Z}Y@<2=2`d7O_095|>tMjyTy_AuYZ~k|F6= zUs*!YD)b>%4RXfe-N1011bLYp&kQ7PRY#9h R-{W(ysjl<5?9sDV{|1*77K;D? literal 0 HcmV?d00001 diff --git a/docs/topics/_images/network_02.png b/docs/topics/_images/network_02.png new file mode 100644 index 0000000000000000000000000000000000000000..5d39ae601ae7c29c8f69fa4709859e0a2006e54f GIT binary patch literal 82702 zcmX_H1z1%F$zlkOt`zknZm8lJ4&Ay1e22?|S{K|Fd^*!Ox!+4~F0uoVBQ$9Te1uu9tt% zaWo%r!62-?n3NFg`a28+BzBuPtp5xNsoL{9*#EGyg(@f+5(Q@=+JhlML%Z*`Cf4>Q zR+dnxaKxlw9R6ioz{=Xi*2MUSJya3W5GNS@@V}$BhI%ij64{$r7(%h!AiX>f+{<4S ztc>iP^lS~Gw5*?BPDl9Pi<{VeH`fEF+CtT~RYii~cQ3;#dbXA(mc~$RxZ8nX4E8_Y zcd-3#2zB-Bd+^`W1gtD93@z=Uu7~1V!PwiE(-ckY%?+XQk0&UhpopNv1o;)6Qw~#| zG~>4*w=1Irhvc}gk&zYg>SY!BtME7`m_CdAW;MSl&UTRAP%(-BJXjHvn-otl97i)L zzo>yn3oUfL4V}j0>6IZvg!C#T+wS=0pj#sSp`~X>J>ATs6)viVA7cAKZo_j z*l*%$LHow;y6jVV{<6X&$53J-qE)B@Me6REg$u_)@Ch8JI>Y_$$Q!kyR<&48Xfa%B zclJSB|92|C2Hd-sWAT-bmW4a#tozHwg3zX^-n>3mwVWf%tf)ZAl@FGrXjGDe|Ie9> z&INk5MY`&w;L0~&-3v)7RB30;eQeP_{`aVxtT&8#4gLRzcPp&ZW76yFaZx=z;^X6E zb}p_JOPklPUqb>vQBmpkM&PG#*u5=QmT-1v=W)HvFquhjR z$r6XRDU#rV1xd(I;7G>9wN8vG=k^mR&@nM9J)d0fuMR})?W>kj1xfg6Au9{-k{I+} z-QC@VH(Xp^ZgwK^k zTP?0#)}JlY3&Z08@9vVE+r(V1UVOOJm>|RV6x-tQz~p-QBPAt8R9bqwmr$k5QNz)Oq0^8<&FXl4$dL1f6KNfl5jNklY(WWY%N^jZ~$pF#jCFM=V2PfVr3fR zkrcg+9w{=)^z4j`kN)4JXG)f^zU*qdUhdNE(tC6HoU}ftDHST~eZe_swsUmc?#9$H zSZ(vJ<*(Q#wJ?`l$&(>mTw20?pa1u7h*j%T{Poeo%CE2e{r!^Zyh|-+)`n1ZHXFhZ zuLVnIE*>vNnT>`LT_Fkz3XE8YegOgN`}?i}6^p?+d3hJ$%3t=6D@S=NOU*g6^7DfZ z56z)`e0)yN&P*7w&Fr>^u(PtVeu-;%PMUUypciS>M(ppK2A*8G^2AWeY>$f&3ehES z+1qXQAwN9Bz`y{T`oYj*w=;r_j6ANZ>cmOk`p5)6k~1+)KxnzSQ|>NyF1C}cP*70? z0zcs6-A^4{<>cmO&K#Vbon<;4Oo#W(#6JElDcN4Ex4T|pWMs^r+UGn$fQNsFh)DMN zb2wNfjkIUw!nmV_+De0d6yAqJW*=YQqm!)cY&Zmjjl$9v{fV4@K|w)w#ElMr1T=JX z=+&()p&vh}P=a&D>q0|9`bI~eOQ(#h#Y9DQXG*jj(o(ryq74V*wI+4sRxpe>M%H`Fo;Nak)WVXnWRIVAfTJt%kii!$LUqxM}B&yG!ySuw{ zrTGwK<>kXP>3X|!E%_$%x9x3h^&hT}!i6F0q4fF;Sie8ndLX`q0S>hqBg1PlHe6vi zm|s+6e{1=4fAx6QOXv=M)aEE?H9CIo40zvYNP}f)Xjr6H`A5zP8yovWx&|$VbUN?w zaIP&i(a~M(IS=YZ=}lzmYsc- zi`8wOpKiGDVtLN{NUzT)O{<4~vxGVx&2u1HO&6&>UQWt*EyU@Us~(o&X=-X(+u2!I zV~+j!QeiSl5kbHc(ydQSO&tPf_iTuHbPPO*0TZ^jx3@!w zgoMQR@hqXHhvUY_>jf)6KfhmTX%X$Z8Q*&&q;P($P`!Ec=5W3`6jR&Tdv#SeBqW60 z^^yuqptG~H^?DavPXxZiN{a_B;s!8(Qa-+P@aUQH1&JJXMlp_e+oS1`-!gRvVn02{ z#k{VZlhoHIsjshpzkk`(#O-;vg{lAjyUFAqspcDv8gsC~?}yU)T1Uu^4oX{%z%ZNC z^P=RjY4K{wbjZQ=^Z2zmQ|OTUa~-9pC5y#oFqmzZ`52$6)mf2(VA8o5OLuhi7= zegpln4E{y8lVpQc^;_1gi5bJ`n9h+#h)y z9v_?IReravw)=8;LEKZPT9W%I)7`(hyFY$bVB+WJcRy{14R3He>&C?ArT_Fv4>-!A z!;IEY3I}Srs-%+A)cfS%r4Q6JG^O=8n;RRQ8bh{pXyy;}2KEa#e<;pTzrE$m?PBmRlNI<;=p^6G7;P6)cWNRQ!+}_@P1o1>j z=uM$=aaM)Az}K%tG%66mn`0+N>(w@4C-y(msp8L&C$$=LY`2?Za(4E*ar9%4hwHOJ z`qtH*otbE{hVSe)>#xnGi};I`v&VnnHGpWk2HcX2jO;Yh>#=h!M}Z6_DmK<2oeYE& zIb~%>1;bV)ya9$X_g0K2rF0c*g<#cF>(0hUIf%UHtj=)Z4I6L0riWnf6MpPxCb2p- zM10OzB)*i<`D(M%D+~Xa7*yfxsKrHYCT8Z~&``56D={%RN|`i+{%G<QE;Iq>grK*hWmNn4(4kpB2#5W z#cZ98K+O6|trdN5Bw_2HMqs69AP0UVBpd~ERgj-gL`fN#m*yIBd+S_dKKISmmZ`Mm zo*IODVqV@fFe!SKb0E?f?*A+3>g~N)3nbL9oMT8HZn~Z~0|S-LXZjQ}X_7`p6fa*h zF)`WP*JZ@O*)`yp)_P(dx82 zGd%pha&8D*uS}J;uA!mg$2f_+f`ZF=KjqziVJTP{QuShtV&%b}aNL6`DUrRx@NWHf zU4~gF60=4=``uqlOI0S5?AEJLgkFz~Ah3Sc@w_8{>+21&6EY#840s+fF|k?m8jRuA z#YIhbcXu#2~Qk0nwXr#{bxuK6WrA$2$ltB z(PDSH7F_$NaMH`7mUS*?>SPgiDz8KU2} zbQ>}wTBdf3hSW6pPj29oE-gN_;!RdS{L)lr6i?Tt#p92QDM}Z^ri?Z+OU0=QDrDsi z*OK`P84}GO-JH;ymuYfl-2CwxWl^silc}Ar__id~fciS{gxDcXw2x^I!f}US8hV`1lH|6<#QS(_F6SjFXqVyzck% z9PaDp?~U)yw;V2?k9JR9LmEXTxw}hSEwjqXJ|+(fYikoM)L4*y{`@|)`y1J`ZuWu3<^cm$%CV6WpqC|1EgocKOx;i_3tIcMjqocoxi^Bt} zc(LQp8Z`{|yRw(AJb_4rM!+2aEC`6?y$|y7dG+E(4UR_x$yP1Fk&#FsLOyN~KI<;l z*}UZ0Bxd6ZfL1y>IuiN3c=h!3!U*|Za?~ks=IGSaVE~`U4z2*aKfOOFW&6=%%kvTu z0nD9?(^iNY-177HpVDaml05u=AU!Q@WGIPQt-=7Vv=zb$68THWTXs7s2RC>yeS|^c zZUdM%I4sOy_&3Wne|oomK|=b@Xu4H-Qym6t^81*6KSgQ_dlH&BaT{CP4s%vEs+aKE z3c07j2@mKDM9Q8y=pP=gMzUsNVq!>=biW)I3BzU%0C&R)7Zw1g0;j9D+o{m+h8K>& z>-R^%1ZNd#H-o=^=TmeCaQFP;0$h3>yqsl}`>#R6#Nqc|dsF7Dq%j=Ya0%kMP0x^r zzAf7qmjo8_VgulDH(%hvE7q3&U_&;0vb) z0^Z|VJ2)6#Y!5r^n}BG$wzl??{s6>JHBNI0P{@-8NY1Ow@kkxOs!qV8sOab{#~)FH z1!v7!2L=bn53a7?z$6a`OqmMNVkpSTc`Fw-Z-4s=)|$)dI42|HCp64ksM8JvdIsYW zyfiO}3xHqhB04W?Sysld?0zBj`1trcM@Kr18$jiEG|EM*zDT@8BqUi85IZNQonXqe zmrzio)fnH=@eSOu-g@m-x)+24iRbd;iu=X;x4xNaEis~GSDVdpUvS=obT>OY`}V5- zTV{q%t0x&HB^y?RdhvF7FTwYd<>qTw=k6~B#7{9vNn-r`(0~eV^+l29{P|Pia$y9X zWVZY}ELfEpZMSfA8s##;`#f*gU~O$}sp#o@^~+C>YF8!-6ezDxR*?Sv+lc24OVqH; zl(*^>`qU$(bg@%{fx7*fc;IB-BUh3{-;|GK@W!mavjl#kYt%BCP16Eu^pZzG#E=0&78qf4TwHfBDkZzyHSLQ}9?e%* zS* z>k38%%p$>Q*`?RBr;1UL8vSLdd|{E+cGA5_XlOzKRzO8X-57gvIQl$q&g%L6MAZT6fPQC7l=c+JoywT!Mjt1nC0c=;Q@2mB1qia+^46f86f5U`t?yd zm9u-4?^&``yBR7dER2ka3B}mh`2KQF0B|e-UV-^WB_~Vh>Jnju`hzDp2ULZ{&F0=$rc;Ul>C z!`I2l$(W7aNW1GpH9%KFwM!|vx$%-Dg=;OB`arzXFW5`GboFR{Y{K3l#IpfTe0{t$ z2$1V?et&>+VQxkSRJHn2MiZ%U*w`k5@u#*cJ=fC+AQAactk}*lMP$=RJ6G}p3Qx~ zn>~a0eIiH9e*a%E@RX#nG1-R;8V4>O9{g(c@Vvadr@870Cx76Wr@*;44iCe|e{@Yv z#ews{Vl6c|6jml%vjB?fFhApQvt-Oh{-fX#;oZAc;3+BWHm|_k0dA%WVD9qr^5*vT zM5Qr3uqVL#&1cKJK@{Ue+yQ3z@|R{eCqQ9<3X{>$@Eh3StFcOBnYQwlAAdd1&d*8c z=sp1G4sHsDynu<%=u!SQA7Ft*?U^X>)Dbb4xCW3eC%;24jGin8*y>@qHYt~_y^ zc>lTaP53W>Q}E`y-aF; zejfx^_i}eUHX#9q#o=47JTfq~Oo2dlF9yuOB{iaGi!2R~{03cF+5p=nzcJ)HrWDlT zq&9zMPe^2;pK=L{Kw#awH(T>KJhP0urDB8A%x{J5Cw&<|ycGBFOpRQ(rtUyXj z8wRinpj*Rt7bZD*#vn(Q33oal6xYr+xiBx(T2YdbbwQdw9Z3{U&5w$kR;#1IzX6sF zDD=kKnjV{RbLvxSTWTj*bQ3-*on(NGDONb=3RUOUAme#>yiY+AyI=T?>F2HEk{uL-xnaz9%7xop-Zr3ljzCG3ZBb6MWNk>UX7Xg?y z6btK8DyPFbpgSLoHBGB=fQLCcvH}yr<8@Eu_kFumWe>O|ad@GoW3`J@X zTzHUpZ;4QX{j#!1a^y{7wp6IluBSGD4%XwjAqg&l&uPC3NQY{%a%1>4I%??r-5d(Q z#qzSUW2^1608+F(9CHDwOa{C-?)wvvUdm`512%_8MtX*a!%0OVUa-hNQx*sWf`$0@ z0qPrFLJWu+AccX4)2y?GW@BRmA_BYPA(oJkkfMr;f|gb|PyvDW=1O2{yKJEchF6jQ z8r+&SR}L(!wynnt7M#fF4+AsS(0~J`PrcgIKQdCLxp3*Y_1W`<*Z<_KjbLH(051iE zE?Yahz42_3)Z|FKHQ?G-t*pu6AXwG>!1w))n*i98PrpHje)&fNf)|vv)vs&TrH@=v zQsVw_RLlK%+W9_!+2!2eEfRtG^U?l(NMj=xv++pP3Wme^rdXh-hIlK`zSeejUi3#7 zpvmCc=P4H+>~V>lFH_mprw|JSd_+cWZ})}8ZwL+wTHQ*}sVSXh^9v0{(AL&|@n~#j zquGOtt9?RDOiZBofH}yYG6h;ocbQ%%n3Tk&@bq-TsHmtHB?=HCHjcwTObq)y%4>19 zj7u7WYIT(%Me+zJjO;Xm_U~fPwqZ)@{+hn+kU%lD42lsK^mCC=bJ7ePR%XWLNRgYI zhFK&}9AXtOERI!H`}*40R`PY-%*-bb(#@c5Yxd5sk*5cIKedv9EuIa~l+MB)Di@-I zIbL6P{m!bP+F)sgxET4OpD($@ABs9J}is$*5xaWOFh0O}HY-V&d*JkSC32JEar zA}Yu>R$D0apj#*#Jgr)bnNNP$PJ9&dl;Ngb;E>7^~)32Is=)Vjx_=B z4FL24D907eEj47e^*%t923~$>2*N60vqGgaY*^_4I0Erz6HI0&2o-yic_4oF0J7-3 zUy$bvc=Lm8p_xK#=1z6pp*YgvtdQ}iupcjq-!l?lh zal+u~`MExL_t&tnY;_ICq6C1d0UH?ggF}_2j(agyK&s2M-5SrW6c+NbpLK%yNJxC* zXjGzOWBrnoab>edOA{SHbpuG7w)KmdEsg5SK($;8IIjUHSdLM zH3zuyQgl0L{j({htM1cQ*J~`SM;peV z)zIMMy25UU^_ZZ{)bv<5o~fp)1|PlDA@1pi9!x%vo&Y4$?FvE$nZHY)5rhce_BNo0 zMD^nkjMUZFpKZp-*sgWF0iwPOV9ZB_&r(ONzP`SI&H(Cz4?qdX2Y_9IuM3A^>7{>M zEP_a{o`|@(aCkkQ!S3ztiRPjJ;b?;PSzCMA*~MkSi;II}O6><~X!mQ_K=$)};NB?l z^v~WvLrilq!WbVP2h{>1lqt=3U}n{=@aSEEfNcTHPu$mg8KA5y%Sk!vxYuvro_C?j zWXOLi`1|*dOggV*8h3rW+xFn#pj_e9aLYy$P{2U}W4u(S6&Lc>mlV*Z{w$%;!|e}F zoHE>JuYr0D3R36i>p=A{uFR#jJ^)gg5m2y!a{W)uaAaUWA1s^sLd{Wpe{^DE-;9oz zR3rhNM*%_@}kmD2~Srn9&51^+TX*@HC!LDm>> z{gEqObSl~Z7Iy#h z<6o|!jIcILu;6*^w57>Hy~PC@O1Ua6+tER; zaCUh+(*53*E1<6G=2@$on|_6bW}$xMpdvno^6sg;ZCEg0=UHnJ#g4RDJ5sx}6~xDN_x5O8&yB8dcSor~d3cMu@N|>34s%;%`mC|8yICG}8I+ntj;PtvaBLN1N)HXORV5gP|8$a7@g52CV$!ljM`sb&h& z)#!gwjVeH6lSQ&fe9u|siptv67fY=O|A{2jJA|5th(A+4>z711PPO8mtRMIA?$1#p8+PiZMEmsfQt)eiXMz=L7<}_!O&1K29 zcU+&JQz?8j$S7qcby6m7_I|A~EG((~tA8^{sQMrfkx@B}f;lL+Pw3A;i%) znrTX$O0tqR`txVHKH&eXR&zdz>Rvx|`E(2`^_Y^eS?#GnZvTlMBON&MqbJ~9u%!3C z>t*DWUya(9S~h(yn<^wcDmG;Z;09clbf9u`^E&Kr=_n&AP3?YW5pNRq8jK1uPYe^L z75&AQ>3*;)(x^d0k&0ZX81T)JS<=*th5O;Gjb!3!A4}(qX*D|s2g~h< zY4x<1-nh;7&1hmXN6hW9maG_*?JNVtY~bV|;(A8vkV9QaG*35HWiLN~^0P}(a(QBi z_tU#_N|u=1Nfmu_`t8U#opRSnWzWY)O2>5G?mKGqBhn#S*WLFY9;$LRoC^niKOq?I zIG!!@VML*#NLV|N;I?phtCap&EXgBKmx|rWwK<(pdb~}SdW^RYbGM-(d8M>@G~d_S z@)YdBg*u-9Xo5<$))u*Pw)a=+@QDZAM34-5dEZD#CAOhfc)8K%)6>P@Q-S6vHisrH z*LW8RPjadn+%$ds9Ybqg4x4zzz?);E@ zhZog5k@vmOD0fX+KK7o=PDf2xD4A;hu3n4aU1HzUg@{q}C8~GlN#PU$o?PLhy2{z$ z4D8wNG(jY;sI}oeohL_pNTd#~rXeTF%yXa)QXDD)YRH6Y*EA1Tovy8b>v}El3aEK^ z&nFwk^wPxv2u~xsqnOA+P>pkbQLNEs7V;-+<=_-47;NDwE2`0bu@##E!|1Q@oAtE*|4lr1!`cZmPy>?^TTN_e1xCi!F#czdEW(v+f4$(hIc?=nuztsUCUfWiGFg*GMVI_1Jpp)!^W-OsXKWSKskH zw1qo-(nS^TCwX^`Qd}C#Pm_6<2RA$AN^RPh{Yvzp`^ewt&LF#<%NXmOUw$R8_jEbf z*ZDjbTp!f5C>KP3GwS3d;FPnsHKhsaA-}|P^A-OJYfej-`mSt?GmUD#k{MM$8MUFf zAeo!vP*gLlW9K4n7WYh*76Hn_(Cd3ED|(=0;|$04)K$cQwiG4G4NyWK-?P73Y;<)n8rmejosy^4dV(&QP1B%~ z;*e?SkK3&a!cgCGB5&Av66_U{b%9U?M&@N(;LoE#BYc5wUS2(QadEkm3LaIAA2E^Q z>Jdl1`>fF085Pd5i`&Uk#WtyP^gid}=zH=m$H^N`PMDkxgDXK5``?%6j`pX~J_R9_ zn9Ni(=yt>9N~q&zbT-xtu75j3nB3gmQo3CSXlTmNDbJ>qX{DctJhv1A0UzcIiBsxY$jpt|e=63_b=2EcnF(jtiOEXJp#yz^K9dVx!x|@9FavOa% z0du?gtB3^2^T}Rn4)1jG$nbzDxkY2-dhkpSD^s0=q|mjAV+pl>o7s1``ePvm4!x(q zkIx>wkegm>JF}hetgadTZFxru!y_1zTnt~jNWM|b^(?bLwvHis=Y@=Hv{8jtW`83L zW#tMwb+Uv+2$IwZtEMOvMwicKWN;aE&K4^enQloJwxX!W$$FJ z{3fQ^95>-DfIT^R5HLB3Vqmxd%jWafNZ%Y1Gcvn`piEAsp|qSAwH%7foDXir zh1?$OuW;9R)CH&MXzf`+2Tw}Tl)hAdKpB!5@IM0AUE;;Xk~m1=q{dAx<1^NR3Vzr3 za|CX@f~H3Zm$YL9#jNrWxd+w8pNOnVouxMupvL!7J^>9dgTIIc@aRTZImMjswi3=x zPhS(U)}R8x2q?s_0NiUkX+1HxKAe*pP7nqqIzUBV%3e-RB4j9ffQ!@8k`WPQ$UQf# zYtrQ^qs~?;4WC3ykP$A5{FV~Wk{pOVhThCg*&G{3qc-X7>uu1NZWgt)B85-Aocl<9VjWH_UPb;7m+eo|uUbfzOlBNcMk1QuEG_2VI zhB2;@c|_kB9<`DY(LB7q>eo3nGQR%&SQCqTZ>*Lid2JE?O>cFt+S`n-B`vrRc%Na) zx%4m5Hb+Zm7q2&$>tZd}hq|E=-(BTakG>Zyax5kEWIo&f5_L+LP+=XzHm>D{x|%7d zuqZfcM5@%T?OfF*9#tOHe87785G2J3A;iAi{m6DsY*)r=I zdd>bCJks9y##1k(Du`CRs-FTO#@A z#kSxfcvqRU&c18u*c7$Vi7OGRjzEHJ3dTRal!}z)yU$OeLaJC6CPHIv?5@>vQB77f z2Q?`fK6KoCR$nGIpB^hLag)$b+b&L!cUHLJc)mT2rPm!6VJf;(VZ+Y?=^OqhvQ|FBpm^!IUvXzqm?(E~znTXnJJ#@w<@2mL+Rig~$kCJ> zn)GQ?)d(3$HnlrI<|bNCnPE3-@AAFU+#n@_*L@^`n7a&i4d!4Cqm1NWtgn@P`;;&?WpZO=#c8qi$F7d+P;XgO{%Yc-nwyfIg2Mk}oT z*KBG={c2L%i-?*#o~ZR>!zG;l^ZHHl)bb)d(k7WcBU2Ju?za&BQUO~jW(k%x7+9N? zVY|;5es0P^Fcs=PgYYx+Rup}o&)4njjV6Z}FA1-QhD2yiz&+&w7tf-o`oIRIS zP1$<)*T1mR>kOQ|+U2}95w7vqGQ%wALf_>Ka6xi_3DaD3BX5PlV%a*kY@!GjdcF+_ zDH{nRV2-9D!sUd3zVhF zxVf9s9NvDpknCuMTv6EVw2ThK(Lg6M=yf=d?al+ew5hqde`p94fnjB2WW>eAt(}~l zoKp>4&g7Im<3c1~;|FpJvN^CAa%@#qOi7}kS$YaO?uW(9`PVQ)D{dJ_CFPUfAA9H1 zOmf8LlKwd)!5qxJ++D%w4WPQx-%{+LLy3^EIbw{rX?~ImD%|Iyks#rp4<8qJ0%8r7 zA8tFuaW+gzlWKZ>a~ioc=(=j*7PWs1f7ceK5YsDa#CVjq9mS32i*7azPwAQftrI!- zw{I?U90S$FSm!YXvBJn}N#7(EDfN1MW+@SC3(-@kQ+iiiLaNib2hC>)Bon{HVXa7W z-|4eF->sWmcZV6CClbG|*WRI>)w>PZp8Gk_9JG%nJYa-+H1DTnX~h|R^E`B&v_Nl# zt}z}Ly2t5=1J#`X!0lvkP{;l0;-%bcwgvM{%UjeHNJ>Vf28YUFd#6lzTWpK((jeO! z#DL8^ml=D^-HjfOxcL*GQ6LY@-Kk_1YV;cjum zWo*yoziVs8y(noF(f$%O3?}D-+Pgq|ZoN%CMi7~xZlcfb6^Ti=iDqs+Ca28e>Rr|h z+v%)yY5NpRi18$BxKkpSLuq537r-0I9U!yjseIBRnnO&eGWQ(*To zHLHR>IaMa_t0+BHcAPOg6}^uX(1ATt=&$s;vOUJNxQEhrzMVc?P{R9cB#jGom!vC4 zZ7qGs-(DZLfBeI*+kDCWRae2uk<7B{L!Ysm7zg8k_~p8GSFe$EyJ~m?clc0ZOPDo$ zUHQGqxcv4~#MpeYg{{iJeKa!u?7g`)Zj1G|OeE8xkNCNVJjx25QT?qI7E3nzX$X{S zS5mjq?qS~35{LAInW7u~1LZKch@PP>_)QxBx+KX^h!Z4i9OX$9Bag#oE#(r?+?URQ;u_(elBmn32#+5RFDHm7JtmvC#1{(NKCPl0B+H_l7fE@de@Zd%7FMY-lr z+h*oxmCl*fIC068Oi6LaH$2h@%Pxq2pX>bnDqSucnYt>s%FnKZ>SWw~8C&ii z%g<#T@82^1F@wsWSWol3R8983`5tNO^eY@SViTX@q0pl%aaNh0yH)+(iNS(QZ!;}w z(q++_^Ij=hDJIw}E?r7<(eJ0FW z#Bb5hFVZn6tQ>8FI^u{_9aGJ2Qn6Fjy3jLp)5(poajY)**>cd7nT z{Shd&j17h=jGCB0z?)C*2>#a`q~*SQq##*iN4ef0g7T3zPr@EEr`k%aW|xZ7PRaE* zd1O%AwUdpz0P<3?$z~NnKhfwb)|CEUiFA=oEnZQ}Do=66I?O*&O7D49l(Qt`b}#Ww zPEG_{=Hapf*|epZ&JSXBL^?*d<&eCh&Os5u z$w@_?X@K3PBzATe3T1R_oZ??#3BK>R;t#xADx=YXQuh6Af!^h8Y-G%kB6q@GX6f&0 z+KQE(<2-2Jly2{8NSQid<2WAw-g|OTRpX`(y-AVgjYNR9iG008ZF9|oQ%@dS-spc zR_Uc@v$o$5hS*)B{Bg&1`u;{yQ0OK@ru4_FSc^T>%HJI!trT!q^E+7h*7XA8WKk*e z$pv;Dbrm;RG@^z(Uehtw#OZ%+k+p`@>2k46ww98DSiKWeG93?PKMo~qSzJ0`bHt%8z5Tkk?Nce~ zA6Uy~Iq-I~`u1^(*W6gN5HqbL9^%j`wdY+a8J>{GWm>;+Fxb4r1zKOoJV!$lA!T+kekO{+Ytb*?DSZ36bErCgbXB_$Nh%##oZP6lxXkzQv|hgIQk$9aXwy-*&dFHufR!1zJRX<`Vz< zr>l7P3wG|!q?mk1cL++(Pr)@|!Aq!srwwG1BTBf1u|H4Pkjg_ECN0hj!HW+z|1uEQ zmv^y0qWop=O)A>jEB-tF-BLXgf*b0E&2x^)LOW=I3;RL6pU5R%{dCs5h^HFVvl z;u-GdN#xxsygMrMYl&%=t*`37*wMLdUJNbIR?7H9CgBcqrGVbme1of6!x>zou&^Oc zzTYdw6pMFth0D6SbRj;7 zXv<(|1m+qZkz7#uo^Ad=@4Q`wrB~%S!Rx! z2G+x*^PF^Kha_54Y3w`dh8_-&OG4(N!Rx6=grqiSI0M~K!e5lSfBYS8dn2zfKlTvy zThCp}6~B6v=I$gs@PMEKKK!-z$n#>=cTIW3)hDD^v*-6nMA2*OmbpbmP{q9PQ`OP~ zTvVeNom{^uJFd$%`An#kj_77?B+n&?JF0z(LPzQx8=$)vC)Tw1$%OT2DGQkhQ$P7t z6e(7m{rIN=T0lU*+)H;yU0of+_Z~#hBar+I&1r|e6N(rcC-EyMtzEIWt=HX;dkKmt zcJKK_i8J=t_~IHh1?h~k=b2t332b_K*$xKE+^5&qMnJakDiwS6FD1HGl~#)V&f_x~ zyGwF?l@)AW4hq51Ze*k}CztZ$PbG>TW=kwB+|!+d66_7vHv`AJ&A>@?{)z{Ne@ ziwT!OBds?zJ)AP^Q1#of@YmW*E)67k@?9!QgwMRM9nW5`*H#~HuUv5EO)N)Pgyp@r zey#|W#|CT}Bw?zlQae!UNb&?UBhzv#cYb}U%&>u9 zZ7(0L|4<+6k^b4xE;9w9aMCWIGyw5e@jscr^!JXQ#J#2NOj1dM@02){5J|UEOS2A1 z_+3|?c^Yb%@yy#Y7u$8PF|YHSgc|(2h9{e&))v?9x%`d40@c}f!Y&;P`(4ew!-$f7 zH{Qqgq?u#o^E;xcV?1V%`shYl_~cE_L@>uS9k2_Ebn6^W(0(Vg)T>CWY4O zTn|6Fe`UfbfndN<6>2S@s7LG9kc=g-8)9?4d1 zT=6%VFb|Wi&sO$_$B|(jzkAnzKXSEPs%#I>Mf7E(^N2`gc(EZZKix@uXrQ)1u3yDo z7IppoIfBn4>#y^XC>wzTY>v0Rjl?>0t=j8PP=)a^Y_fc26I}k%G&83?g@bZgy?pvm z8w3Y_G^~%Lg>!YL&W-x~>e7<^npqV>nfmqr#?V4VjK9~Ne-rAy~rGGSk zg~yp3D32o`O2dYB{pb?6^YCcsp!3`5C;bX-m(r`u8)-e)n-5jjTHPvk+q&Jsb&Lu> z4)%n8X5Ytv>mcBFtw1pqp!rJ%NKy-tM9osd$rn_4TdfQ`KWxhoYd*`G zCR^LT)!^7Lh9uV8$*lclL|uaY8kT$est|K@Con5i@clieh1S|YgHiJ}$N60Axd{5m zPvqsI`jls&K?XoC^ZroqT1}OiUzaN;7cI(>I#C_*YE64%SCb9@PXFJ(w?ln#SA%wt zPA4s{$=_i{+_p9o(S-qzGH7o&~anj$~41%n*LVw$710tM72JR?Yfw<$U`({>+``w|@oj8eKD>Kj3iJ5TZCsW9bI z5waN4iCttjcGBqs#yK*osjK1s$0uCQDHDDT_3*AGbbpVt9!ZCS5A}_MTL%oKR|_GT z@{wIa;b=5(!!ZiwgEbAZDvF|M1mxqA+2}%yN`x1Ne$W)=Pve#clToE=+Q@~D$0^Gi zvid4|gC<1kBy=e~lklV%&G5ib)Cvk6LL+b=?+@4|$8>#NEUgCtEB5uD1H=cUXVjT5lc`TCw+zRq1Uo(T6Z;rLZheuzb{JIaw`^W~K$|FM$o%oD zUAL3ulK~pCuUuqw(x(;0r}D4*>z+bpUHMa(4kZu&?FGnSOZrv8*yE1uclmCV&m}A* z()}mb{<&xmsUInv!B?gQMrxamax~pGHQmOdq(5_a+p8iQD#yoKH>R$6tIZy2)Gb0~y5xW$M!j(hIm0gHEw;vz(8rF}pZCz$* z#4i>74Bpmw?XkW}eUj6ob^|hE>o>0{N3O<_`pWmk-k>rccIKXGAC)g0ZO@)<6scF^ zsKuZ8ueFu)TDBB~Oet~nsbR~ZaoFE}Otx$=vE4p1_h3z9MvWH}f_syNx)X#|w25Qf z6d7}MWbT0(bho~1{&szwA=uz;xIz2cmv(yyJLMn@k=--|O1;t-FXl^ji)s1JmgMY= zISP0i3iQp}&<8k9+`BdpCBa8buNCPuP9f@~t{+Uv9CpvyO|H}aP&Qq#*6%E#`(sks|6N96F^JxU1-5 zYBVdsW9#bZf-$=-ou}#Qb+UNEyy(M0(LFEovr$*X-B~vpqB70B!8!N&NXd0=nx)jwKJ5C@vqf5)B(Jak`UkB=y!{ue+A(wnd;IlquJth~7 zKs?xBw0m`raA{?1Yw4x8u3m*h9P>3h{hwSvij4H72<2vaosM+xyRJ4pwqetSiM=mT zq6YfIMV`l09C6}0^F(WWkPS+(=_dd*0}a7F+|N8GC1}q-rR@(p{mQW2u=(Wc=WksP ze%P+@`-BYGlK7TBAQorKbv)&9>`t8SQhKHbxKA1BSZSL7I^EW=DmNPLQqFmW8tQ;9 z#Gl5f;R&{t)L1PzYTx*6iNueIBH_bq_%(gUbF1~gYP|_uRMW8*8_}nfN$uFOmDARS zq}|}NTmO{t+E)lAJ!csTpIV6te$cdcxIqf)y) z#v`$D@!A5ZUz6&r{7SAZJ3f)()_BAgC$N8oqN1WA!U@*=B7?Jku)?gl@$4ozzUWmkfbeXJWQB>~0H`M`pxN_AnSt@(z=ZV2ybJldH1#PhHKQ8y1 zMWj**ndwu5XWb?}9D<=9LZp%n(%g zh}x~~pn>4-uE8z1ySpd2dvJGmcMBFAf;$9faCdii*SGVW?>Xnkw+pMGrmClB_nzJB zUiY==R6}Gb1Ig(}X}w8Us3QXY{m&p{In20|W$3zZNdqo<%WER_&7v=Bge&Rlm`o}` z6mUc;kK_S8u>XdG4s!UwpESn3*R>l~ZuZX2&Lv@!!{eiZ##V>*tGXq+QYYMkZl>zT zQq>4awh&gc_cnsx%4lXpM@}~*N_-$&xhJ*@C=sZR9(tb_$k|%qctvioY?VytkJbK@14sYNq1&_ny zef{t`S7wv6SbhTruy&({_bEr(&o!TahW(qL{mZlC*ZyEbptib3%K!gaBMTo-Co=Rs z367haV2Jpme=?HVZkI9lk&H&!UO56famkO<%t|v|^|qrYm6pNhrC878B4*+;ITUwN z&L>dn>G^RFZDX+GS&08lAFa{Rfh|)SIo6bUaZJ*XNZ}1MV*NZgp=j5VW%k+a<$D*| z&y%zEt{SHr^j69%r_O;y-#3xgGgCJ3RRAYbYx~Nyrdnr}8f*qM`o5~A8@jyzZaIo) zyqP-Nr%#Fj}o6BwY_qsV-?P&aYHLWAPi!OA5%*cE7O&9M|1pC#anSiWOx=ks? zch&ok7owZqraJNfX5&BYF0lir!a$qx^fQyC*di7HpzbYFMDq4vW?L} zq8I9uh+8;$0X)L}{*PC%uG8wxNZ)HFD>$qX#g@mjTzK1n!% z*P|KyBkY1dr(paKSR7N&bymInL+Rusxv`q!J;$Tnmd%|9I(T?^W*@(X$n!}8q~h^I zp|VYXJ4$ExjmR-&cjtWf>op`LxZ8v)F1}#mfvdq|n98Hb{Cg0s@3dc&Ha4}VtgOs( z&?L(n#)wW97z9dTUDKs;2r(a;NE3*VrzNqt({yw|HX;$OAt&Jq;_odwVNKtF)e4<6 z6n4b$^2f}Y6QTcnBxil#>w6)ySa4SFp6Ep26U;hgc>X>)ws^oxBDIk7@I|kRy3mMX z7qO!!{z}dIYZkJX*CTc)qGw6zumk^vnBE0X`|B;|O6Q{x4LnP!dBQ1gPPP=el$qLx zq}aT2%N7a$dq0vgbI08^L>&|Iqmo`x@Jo|9VTy>swDyP4)LT*&VNw%zM!s9@qHaa1Qh6)H+8` zvisnP?>~~DT&KhcMDqjP-l7*B)m1QgdaKu0C$8#)sc6i-0e>fXS(s8^PK{dr#gtb}wV{^=DUZ;!OR`N4zY~8UL0e zU7zYw{An=xj+R(gZizU2*4yZZn-iiA(fU4EfAskxRj;LlfwgIW9>B-hiX%X^cItWm zP5z};b>7b8C5C3-w{SFCn#9%H5R37QD}c5!0;5ON+X-S1uc@Y>h7i2NiI9LKAa}eI z1^42ENqTSu-}*s?Q>)A9Tz%c^g#MW~uItW#P}i)`>;h+du3XdLa?1Tzs?cjxl$)04 z#9&M1HEw3I{0rB!1IONjZ^HKoN?+C#6^$=PLyvSX507VatjAVRT6!0SZ{Ni(Cmm1u z&nT*Eb8J?clsL0DTC7()qX$C3Q#O{ev|b^KkKm3pTUrU>P_&oZB$pge){jEIsV}c{ zekr%x&-p%YxKM=6=^Vy^X`BTw% zh3~W^L}db>tOYy<|A^R3Sx?6@D6mbha1E*4^_=*=yor+wS%mAAL9D3~Y9buxtlwoA z``~t+J%p*wMFPVgOJQehF7bOq%0L-s7f&&NALR<3dMdJ<@z>U7(h)cz(ehZ9V|%pN zg}B5zT-F+k7k$UB?i-0!w1=wo?wu9d6s5QD8J9~r8lpEme$B~&g|6x0M+iL5mEe2F z%R$epeetj~^(pJK?Y)#!WA7T2iI@reD=P?n$D4hSp7ChI z!=ksadasR+-9)qz$Bp&}!_)#FdcGh9B&0mnJ;mDmfa_Xq+pvjfbE1xp7~y(n zc`7*sgbxT)4!;6_ViFIPV#RJ(B(xl`|LLv!H^LW@kB7 zZq5&9fhJMb-L8>S8sykCsNCxKge^o6OvWAN=TZ!&!ymy3vIv7(E%_MV-H6D@j|A#N zUA4VbtL^etPqrG0H+r(KFZVg#F%&G&oVmE4dl_bI7a$?P82C zU7V-ak4eArMPlynR<9dN(&W3peVM)@Y+WraLVwN>vun_wj762ps*D}OqrqN#!L)OY zLVIw&UdiQGaODoYD}sIB)9aM`{Ebav!EQ3+ro_fPzojL~pQuaJju$k^Lv`d3Ott-C z%g8UnHC6=c^BgGRmZE($LV-6}Oj|%|S9LPfa)04u?Bbo>=zEUoGx>FTbyZZv2FkQD zl%wkk)xjHWtepNf3UMovZqUvcT$}{-T}Pz3S1MtPIi&SI9(~&B)F*?e>!{dN1gi7x zGw0+P1=H$4Jf8^qzaw!9!%~n)(e{@i7alw>=E9^eY@GJzDh_bQ%g{`xoq6jqSS<~| z%ym7n(A5z(Tzvf=!VT4&wVn?8;e=vkk4*pf^!_M)cZ_!P^ho? z5`?eZc{*P+SO7%3y&EI{!&fKikl8V)*Y$f#X@%Ij-@no8sy<7|Lu8(>4TyUpxV~;5 z+&-a(1+p|=%UO3i(?_gG-k+_x4LWc!toBH3bcOTg1x)%Y1;|vs*8UZod-im@{bxCz~)4a}O#Ur;wRh|*XtYkr8#)T9{R z9sEddBK7%+T~f9*WaGLjot)>Fnd_c27tK-Aogkpp1)-HSrKJ0`R)>FvFOJHAoISVj zO;N2i#O8Zu3Cr%!TMfponA2U}x3j%S;{3apE3rh!fTU*M#N;otBAcu} zOv>ZlKsk_ojK2T$ldK5l^VB1sZzPhESGnCH_;auQ^^_&UR!71{K1A~1tIujUZ0UFX`-^Qq81xyMfA)d9!;kab84FmS-{HsRO zS9*xY$m7M`$(^S?Vt-M-c?`aG?&fg%xezPUOwdZBZAZmRxvNh^SE=0rmVi$v*aZR+>p$a0opo%@K@AUzP z{cE?~&B_PwIMEx3-JzIP2t`c|f$dvm_c^zu)su_u8n)?fs6YWDr>E&gfsOO%qGN;xXV;Wso-)vuctAD!AU2C=zOJbHqp0;~zaJ&q(S#w2|hAD}mkK@3hsmUJGa_+z$k# zrpqxih26xDF#TXg0e%yHQ+ZI3K>wE%8L2tjAtVB>l3(dF-;rZh(-w2*ojYIG>%~OQ zO;q?JZZr%of_BnP)tT1(;my))(&oYEhnli&lwo>7?jg5TNvD`Mw76TX+ABycoCFjB z-A^}`{BHsu@#7kuV`u`HZiibuOfU_vQyTme_pc`W%@~g6?JGmBh_0kR(G6~Pu*Haw zNl;T!cCo93VApWz^XKJ7W_nj2f%heX3w(MQN$uGLd(-3(T3QTa=9Zc zz_44p@D9xCVdWG>=Goh8k>n=BEKWrID#MZj%@Wm!UoZ&Pv^$}bLQuFOT z@ZdLUlaE0xm~7m#rUWiaQS_*!>Hc+*sBGq<2m!>z78K-_LDXSkJ*$mMaZZx?e$=`x zE{@%N7k^I3C(h2vt7KU5Yi{CWF^$fw3?BO;UtjQC2^mL*&=CBEk7U5LnR}COYMw_`*3QxcAJWox3rC)a3|YY5nWB1y;X1 zqOBgmX4{5Wg0soy{Xj{pHjVCyx#MCpU+l-?8~g^Jh5sss^+rwhx8Q|dPK$o^CeS)w4%t0+RtL#G zcL0cmDW!EU#*>b&?NU4b@HMcsZ0q(jTlK~924C(7%j>JIpMJi+`3O@v(0obh%xAj`rKo>{5LX)BYuxpv#HTfTQRi z0(qbEaf@n8Z)2Q1DMk}KbcS!HaQO7CW7Q@_BVF1Wkj2_oYl+_t_C-YZ^FBmgOrp*Z zZ+K6bRKUVITpwY%k7f5Mr3YT&rsDvD@#2{o&x5}|+3EY>);nIi& z?&IVc+3Rk)x3Q7^c8c1n-npMs(0M#|QNIzjL-8{z@^dZ$`9pg`Re*nv|Morkg`h@zAq#6_=XoI+w$7;yoZc)~dExTuV<- z_@;`a-n!M^{#FhRJB_+4{l4TSeQ-(T#+kov3MR1)0*C(txgSK`*yBJGaXkr(6R^_@ zkN6@9_Pdt*_{)CR;`wj~5$oAo%K~sVMQ+=?hW5R#={@kD*oCuIw#;q@vs`B8FCeTd zN@xeAtyA8lgG`&StZu$x`4Ek(5Wyf#&0qAUo%Eq9{i!VDed%NX*I0iy4)<7vM{Evf zRwF7rdPRGyy!E8A`-W9xYr`oY^1H61LvmKajX1=4@98u< zTG0$i#A1lH`gad=Qz)P~%PR@J2kgd*8flK++FkMjKd98Q(Hdo|(X^6a7h26bJ2 zTvPNa%B-R<=a6sWCo5jqToy9zg$E!yc5V{qmOxFpJEqBq(;M0OB@9Gz##!-&;iToz zt{=!A-@)XIdw#k`b z@p!UNY#iUf;^cyW$=rt}4~APYZa6F>7v&mlR9@L9pEOQF^b`!mDeEDb+wgHj)luVotLwDq$>M zNzL;gC)wNm;$yK^k>z}n%~Q=&*REH0YLbJ!kD&@TBO3Xy?`9W0U%@rZ;iep0riL#p z9SaUFgm3lCKYMc_qQ2i>yr*=1v~_juJ}z0OsB%Y7F?E5)RB6P>aWl_l(BfBo~eD*z_IPu=ruj9BmM>}zx5 z(_a&&3F(PxL^GB*j*%dxi`OU{dL>&>fw`T!HF-! z$$VAYquc@)fawD?ceTA%B67)uAab1tOI;RApC8{fE02$`H4*@s9!pOT!JXD_FEO|4 zE^EVa@m1o1@w#r`p&)rhW(0tVHPpPcQ#-VN9@DY8NOX$M&8sx)%E@q+OOzT&bp zd`Gry3P=(9=G#Gg!9#J9lvC2v_^t)2yh1MN4_!r7RScaUZ@xR#AJ5gP+L%BI=-)`T z%8?`oc*LEL(O81&zyAC*sG^5UCP5FUmkhCHm!U}>Fwfe^2r>rFIx5A2`+M&)L$Ipl zcHy^dTc7VRQi*|)!9&QgRFg;hRZ^E3&>$Us96gId0D1%fdV;BSX$Rc_62^Z9ot~)D z55+Fayelj+1R-QhbUmdcC=e5dYu*#*%||(bFU|SWt)}_AE<&FyrQ-AM^E7`{E5R)j$l2e5lmOk^7sBX`&Mp zw_DXo(<5ppg}Q>+-m{nzde0UUe{QTIwJ3Y07|2)>={BB#D3g%xzpwBILjnLxG0)qu>sHODp`<34bHDkSTV^e@NQoPQlUor0LU#!#NvUQ*&N>y0n-4 z;Vj7Bt*AgGn z#!II`bW87 zHnv%%GbWP3USlbybv!CxH39?#=Oj*ePU&NF^O7D^X)>-6O`Y#zF-p!Wf=tSqI#GM# z=GfI>!g({SBCj?Zl*B6tTm4_PDLe1cz`|cgO>mF7DN1%>k#!vtT z7_Uj!c*sIesdRx{{NRe*`Zs0qNE6vLH*}XGy%Dl%-XQ?y_U2}vt~z;|k5v};cspC&j3a%cId zmJ1PXOcMM_#m3f{kYx4Ndne?U9&P|q{DXTMV2=m@45g>*)GppPqxhDvnH-f@vz3X* z$gz)L9-8ybze7TVCEjsOp&O=5Y=1*f1Z11K;1o+7=Wm=|7)$=BUrr8(9>U$DPS7Qe z`z08(H=%VcNsD+Rs8{+avzTmkNNU49Q4}S~XV)3f^e3T_SrSsD=t)Y&8!_sUJ zqqMiO%tT?t(5Ul+=FSCWI~JS~J)D2v)n0mZ098zO>qS3xbSalcG@m2)!fj>ihxJ(V zhHO_kVU~1qYTTd^3E8thg*Cn(YHN zLu#ZTao9}yKa4UqJXl=fjvz85@HVH*VwTUbyd|^XBeJrv1PtyJ{ZR}AsPo7$aUkQV z6K732OaR5{s6*f<~ovVj`dK#jA#1v4dzt5mD+iaSMZLHdqd1AiDz=>&;`heL6EW#B5`?i(2> zOo>5-8j~;_(4QVE?^Av93j3Bye{h@_D-@1~C?RPoSi8H)Cgms0f=Y}%LVScEOHi08 zCw>KGsLy0nhaC?Ytja6RnLr!~X6BD!A?{w3cq5>Aq-}g=^RVO96g_oQLXUHnlz~r; z1Gj#zTdxJhh0=|7PkLPWw2WVfUEYQe0dKXsiaa{)6V-_u3JUijy8OPg0B}gITwblkz)g*|?g#{p8U}rQP6L{v8Ta@btUog2;SpHvt zkq+Q_O*T5;!ty1ZoiS)N=BMZ8#Psw8mMTVDovo&h>H!umEdW^t)>%p5VG)tuLS?B& z+cjaK?mYrowENrI4|5(EZs~<;inZynI4ZejRwOZtg6c!v^D?$LT!G$oepVjZV(yc# zyl0j87q}FalAL8HOPZwy7w>_@*hA&E`Z|^2syz~RBRJS6W}_^_%P5bqg!oG z3~nJ`6htjNHV`9%eSBVSY{8q*mGv0~?QTiLUi$`kMKMlJA5*{^SLgkB9Wr(*$wU`8 zYa;!!dS*Bz(;RHu%AxDX1ue7A^-(3p#;i+6vOOyuv>*e8g!96eJRdTv6Iprv@=C%s z%f2;Bh*M)g0(+o?J{iiysMuIXfM1#bGUjyreHSnxSu_oxErXDJAC=&t`6OcS`%ms1 zS8Tp>dOgBqvabc@Q{!_vL;-ASaPYUcK!6BeuzdxzQY;;X6I7 z*@|oOE46b9`8O{uXq0%ZiZqKr2cGJ$0qRvy2fB4d@jPL7rz2j&4G;Bgd!JWtv-q9$ zsymW9GzyE-a=wAn6N2@UX1G+;M5k__nI7!^Aw*oQQT;w~{~!nv0ZkGMjejZjM`!qh z-AHB>4kSAK(>Loi#r~Q#v&#@PP3*z~8+17_!sk0y%t!sm|T1HwQi^Hv1hZ9ZP`r2E;c@V5cH$XfnBpv zKf`s@6FIBY7kaVw)fd#kRf*;zrxDar2QWy4WjbXV`bBI0b=TI*=W@kkJ@aZ-uNv%NsbNse1Avw){KqyQ_~bv>ufqQBzS` z;c?l7dZ5At{mG0esYEhX8$>*gRZ-*mz+zvt&3&GW4DXH;UFVu*6Mz8)GOb&bfCinM ziE3!e>(Ak?g3xR8fBfta3I#?xiAhD=$2B$(D4p@ru1$@BE=cMK2 z5dN+5eoVpg#UjNszAGBt9te?=&Q-^uoX3?`mjHSrCT@eQU~69*Ux=BR1-w@DK(H<+uU?kL0Q-LbLcB#gf==f2DyAepH;`3Sz&yjJtly zo~hf@6U}{TD<>TiW7b{x+qi{=V8zaWnvlCF$^@)8R8#g$90KQ=r6n;94XmrjzH=ik zE<|Ik?mb+SpFgob1q#K^903H{gPV2DzFuD<7Jq29ol!;(n8yrXMAbV-Pd+7 zh9;&yl#yLQLra;MfFg>*g*Si~7B?=vXyt&zJtIDE{MR^(xudj3Ic>*ivFO*7q`4@H zPzvanJ}F66KUtI|{iZS^nd!SIY7Ns-6o5t+N|N1OM2m{0qMDvIM}|Vs7Rm4%lmR3J zpG$F1Fm4f~wQPpfb7sTz@G}Z@PuOT$hby2rv|41ubMF@4TUqOM3DTkfIvbtj^fM$6 z#T=|h8?=4f;3c$hA>$u?Ewfr;SbkC^gIYSLG**-l5AH)ELZiaAz^(GKt!dP&*d4fF z*ieY5mx7@}9Ih({iI`hdnCG8ugFlg?Su%|B{;H2==f2S8K0kh5(rt?wYhzb&8Rrce zS0Ajz`yvIOTR2D{E>4Mov_pW`%Qr2-sS-W+hpn}&q-0b~MNjX;wU(~Vpcnccp+k%b zr?0HZATDY*{kpVXg9L^NAIFq;-=M+qn|xqmhlm{$HX(7c90`nKW*urY@%7Ant}K<$ z_a*kz=c1etRAdBp^(365lha>zf5oPgL~L}vxpc^yTc;U3P%|u0Gwbl1w1^8*W)I~% za{tkDwL(X)Ev3t!idwT3PGU92ppOye;UeF{4#BL0AvDd(5BGdXR0|%8De00dA0%mzSAw5u&1+HJfbd0cLRSgoJ_u zqr4d+Kxod-7q+x40|=Npz#9vI%7ADK*aUKv*g;sxfPcvg>gwWFRy+);aAN6?!k6c5 zRItH46g8f7CP3py&C;optE*Anl4eV7N_V%=^UG(@$VgFVXN)G@z`rf=&C|aa z9S)9E0D~)^Yuml&U{Is^8%q@n6BEF}U%ItN#l$3}rb_i`YDtjJ%QWfbQ&0Fy1qm7>e?Rhw7)6zKDeJ^i zY-J&uC{4U=7PqG$kBEXrB;pHPLF&h#r5>G&kD*x{xnYU`%!imV?kG_1i1o0HI&0qW zo`L%G26JVk#u|*3Zz8}xaFnYb{Q%1R1Ruf*3@WAHU4%Rkr4Ai?&1!IlhYTDZ9Sth#PJBtUnAT-zxYx*e zPqdrCl1>DP;>22&GK>gGnV>|A14cyodc;lK31zjmgll0qTp>YZbQTcZ^C~`m-SPxo zX^HUqu^Q*zCxUZw6DUgXbAu3Z3HxX%>vmVx-~}loV9zymw0}{J6)sw%J&M(J^<{YT z4r<7G{4K((lsZC~owi_Z+%8WF&!U{R8leGjW^ z3XS692snCA_-ABBCiH!^-)mg4-7zA1!PeusRr3hsi%j^VKKoC@$zN6Pq&@RD%K{k%O}p%Z$? zeFQLLOMbYM##5(2IbG`%7@=e7tmBsVr{oosr>Zrm-NXpYfjDpv9^9$Zr2+gp7Mo(a zwj<9*cf!KKXU#bHD>v|Lc^qoi>g+M+Ta3A#LZ|kdbB)g=hfu6rq zxvf?~er=lh^%tgZ8v%_5^psLEPh1S*0mO9X`;sOR@PQI8j{1LvZn87A3d6nrKlKyZ zLSNtSUr-dI@6$+o^|7vC;hblPC?#_>q?k})k9v1Z#ELy&>ZGlV7)FRu)eKVAg!DTH z|Bj|hll!_mh$T{Zf{tJk{cx2h{>dNjZymr?{b!WN1WpVgK+OM4X-aZY+8OFucA;fh z1L$VsgHMQ(-w`czd|^tK3jX6zCnM|W+XhJX|2IB?cCP*f$jmG*HqX}nue^BEw|xcW z9UV2eBRa9va&9>iCrOS;F7i(e9w#2TSs4JjCKUF2Ul*2&C?^nFC!d6{j4V&i{%OJF zQpH7FaH%s#w*e!M(qzfW$yb-XNMs{ei=5j3Ibz_)UP3xI8(@l04(`{3$3Yj2eqPU0WVH06z5yz-&*|M*P3SPBsaE8Sy3m&#U?$h1KCM z%~>As?6)sIfb(_RqAoqz_?MxG4%1<*s<%dys>HVj0y3_esU;cs-P*st0bryv{u9N3 zEJ`-L#~v)VybJex_|Do*(a&UoQT^gil50lCTCg-U8Zcr?Kv;lrotCJQXz>gQfYE*X zHAkv#b1W$IKd;C{S5m$>5eTe+n8Y~Lt`GW53em^Pte*~#x=T%_6Bsq9@~ci!SzS6u z&ERqk@inn&H)+8z+==;608g%fRasR@BuDE9Y7JH{osykjS_K)^M1)i214%NxL2N2E zr9<^+Zw3@GB^XHX<~7>{5IKO_1z^jvV2t73cE3t1#5TO(0@V3v+?b{M10dURtczN( z87xszqg)>U%%}WZON%`TLEjJ8Q|cA!}+Heyj6IV=}03I3OO z3h9k_-TeF@aw&62Fd;G$DVEN0A@N=da;ZQ!9cNyX9x{fIkhj7E9bTzbdOr9kqXXCx zWPIFw-@h`)H1y8E&gbfxrZ5sUvL724R~YzQ`tk)pzHj1CT|LuM*9QDw1Lb0*oZ=VhB>7k+ZKZ!w{TorXQ*X~jk9ej<%yln>vit*W zqV#;&%YUH4hQzvTcZ}Ayv|o4KAMrQDid?wl*Z@&t+ciwEO2X1n4wsk2HO5hjEa4UL zVbOQ?wa!ty#;>xZqmsUO{z+lThx>}fp>99Z*7FRMBSK25b5rt&%_6tF&Qx#}JU>J| zE0kd1W0_cZne~CjmX=ycAla+^ZvOkK!wCJ1fci46lyO4)d{<~}^%HrXY8VhT&{Dk* z(Oh5z?3tT8wh=ub);?L&cSJ9I*6q!k*y_wcx;!)55H(2GR1%{+JT{_3n6sLXvzjF? zo-qbPAaNS_p+3u@NGG?RJ0mD>0sm$m;ig9KTy#%?{{AJOdT5DGOH2?MN>nKAQ7%R4 zlMpfr;1n@2#H;@*;Ho65I1MQ=5cZt_eI*(QyCPLcY9<~Q?cnFt!fTV-tBV%LI#G!2?Kr>2bvU@0n~ zCjINF8#ywoLj`ECvXrIO)r0122?PWrEguNG6#+-*4|!kda)!zs)0U^@ks;B=gwVw( zYc1#JJJr{BDIi`rLvQ8oZmaD zQeVhHEzjnZd$~h0kSJrKVEIFdF0#nKot2ZqYtw5b(8SnHE=-f6Sym9&snIp6#m1wM z)1QUN494hPlC|hcs>>=St)#Er_69VA=-Xj3Ydu-Nu=r=NlB>O#l2~6dWHxKkj!jKcCo`F z`f4gIyVrVUn0Q!TaM;MXV z7G*cFK&NH+ZV+?9A55EFWZHXHQOzFj)O7irgP5-j{oS5GPR`2SlGG`evnM_0(^nEH z=b4TqW=SgrJbn$L^D);NU5>sfG#gQ6++z+77MD{<2=hb9-nV3&n2V!6J4XGr%Dc%Y z@X(i9@+geemN!a#5Xt(Qz$X`|hezrbga?NcZmL$0I4}3SH@lZpSUUpGA1H2fB5V=5 zr5+KUk*gew!Q=*|ruES&A_8Jvt8NZ9HXPltrRr&x_bnIs9L{JS8pW0;MBEA?#7=K) zy&sP2Cz<6b;cct%Ea||;!Ykh~X)NLNjI!;*x%LRY=Y>cVM9X>c32YNkf&qmWxSr+B z%&>rDIGBzO@4g*aK;H%I5)QYA_AY0w*;7+WK(R?<`l-xns~u@l5m#2ROu8oFx7~kzu$tC1c>u4v~im zTqT;ZIAah65PO9D0tJ6=vr#qfEY{F#HAdLnHPW;tsB7EUvDRG>B@nDQWbF2l=1lmu zvEGc{@Ul*f-i-22u+pZ)Cm8OO7Uz)QMk^~R0TCHD_HAof{A+^bs3!dpdqCJB5S#W1 zxa0U{WqEjbnBUZd)v^9J1mQ$duAwWh9X{qx*Qs5Dn1D}SMl=70LbKwqN5cco%JCbF z5k2qvZ)2IIC!}MTpb#GSt4SO$DKeYBgP?o`P%dcU3%7%B98p{;a3$ksa%2KHBP3e zDvaSUV-Jfno?L>cY>wS)hEirWbrc5VvNk8}BkVlS*A8#6nTuAb{CC?JzK!q(Q=G5l zFj3Ll3A6Ats*4V+OhX8e>8EnVghWKeot-%VL)!vw4v<*L+vLurXKcMNnzaGEtkqNKtQZ3A~3OWRDMm|oK969!Qco&P} z8J1D4W|mUcOOnnOQA6$0n5Lx`HjxR+o&GjAwNs@~tKBAr;p+SJG=lgxpX-t_GE)+a z)KM9#^OQNFiq}RYsl_9-N$CVZ+utf`BuadLW4b=cq5o(`(yMPD=+7mx8L&TLIM+Vk zKa%jo#<*#D)Qwk^EL5BSxifrSx<|2_B~X|jd#@t@g`*>ItPvz2k*y<%1Ay!5;mdWo z&K`d3KD=DLynVVQatHJDMS>0tpNzn0{$YzRH8eJs-KfJ$Slrf@xnj2=LK^?8t?dBg zfdmE|h?_H+$N^$!5oYIejE(zxfzZX0YJss<=R`ZgljCC`(}y`c=GA*Bpo#k_qW6L)t`{ax6S_f#XmLITi$hUKW^{Tk&WiTD%71`P6kPkG_&UmN}XL)o}+hEr7YF7 zSB&5Q&AE0eb3;+C52+|ZuytEOSX-x(=YPEb&j%|wu;P<53JR`4qt^~m@g(E-&x#al zaqO{cnUS#d1UvN~85FMLVRD>Y?k;KhMQ~ic5u)NN7C2$b9du{HMS4~&c`WraKQjMj zkunp0i>C}XHm0@rd@aFm2?9hyQ#`EP{djvSn|N{{L^RF}ab^j#cjuIZ-&C+%PUu3C znEY;G44WwkUcc5KE?I>*I6 zCaDR(Cc{TQ?U{fenQt}Wwd?oUB5GBbmv6L^%S|>{VUJfnhkem!GLB2M5p#d{LQjN| znly($bk?>`7em)fBu}7;;wI1z#F0?H&tz8$K#uiG3fc1te!jlO^QpY%ZIYFVMIW5m zh#esiFKWD{fCH!IR3*VGBZlL>^-H5 zsj=}N`<(%qcRGcyX0Au{yc-Yr*=PjR7}D|x3utIYPM&Q{C35J?5pFdfOrcqy!@`+c z+p}kPE4r~Ve_7OT;ZhuYLD!zHtCWT3wD$l{#GpIc^CG4r33p-Lem zy1{;SWJ;;M)VZ^%Xfk;u4l~P4zbe|)|5l96C^d|#1h$O-gnOT%0*6_RC+tcs2%9TW< zHZ3lG70rpMrb|g%as-|)2Jcid7bd7^i))3XF#B3BUvPa*@8dnD>}bPC?hdQ=fF+|w z(c38q1L8hXN5&qTA9^Yzf$9%Rva}d4;ouGqgkbSZzIr*3Swn#uKDltvXU-;j#ve`{ z0W7bxmKZK=*%1*Az`8tYiUwq88ygp*!9ZTR4FFY6>Tpr;U}1mi2p~qOUK@dM0{hA> zE01I$_!7u*{mF@sWoli!WDR^iKnkVZPYntZ2WeOSE-b_@QVyn;=FEn`=V9j(^tf2} z2I}*UH#o~HJXL8_8r-zQnO%0V;;Pm@T(Q zWqMYvLj(bPpJvh`h8eVCzS>toBl7A$NfCf))7p9DEBI~(? z2S$INuvamgo7X82K}{hgzh)OC8epyp=`=dEWyrF$5>ZxIxN8L%uqt#rGK{KHF9+Xt zwXdn=zZP*81!ZJ?!~Q0|Jh*}#8CBDw52t3K#=Fs!n^IF3gi1ok4R>bR2Y( zj8E~GZZIYp*{U=6o~$fZQieic%)3RUMPS8L@_QM{#)o&t6LmA!yN+4b1OuwfvBR%y zQIv4Fd)Rl3^V74KMprggAZ4pbO9@WTXEyJ`yr173HqFVrjkGFszlUiN^es7OsGynB zu-1!`ez9IEJ&Y>d!;2+w|^ewwy-`fcpb3W#SsIyB}(9fD%W;% z_cQ^|<%wwEgey=c-b6H{cKL-6ioync`v184$M`zFaD5zZ(%80br?G9@+Oh4%wr#6% z<3^30G-_-&cJj>k{LXn^{NE+}vuE~bty%YVU#QX*9bCw$c?T*1E389 zIeb*LZ<1R&`TS1FMD0P(F>o?STn0Dm4w-tl=ai6Oyt;Fo1(GTGQJ(jbk|lHSAzeXL z)UOMC$v_BntR%Wr0+B?KG7B%Stoywm5d8<>UoS6enevhV^~Bq|1L$yFpEu9UZElJH ziBe%<&%nUo&j(6w19J-tAn|GtNUsC3?tyWJ5<|)D_3uAE9gqYE0}P_&GfHY|{E|6o zgAH1MsimQDYh`Cw-q;8WM9UXT!2f&w%KyB-fPjFYs;w=psfh(dlo9aWfip7J;UN6S z<|CX0)$CgikXc2QC65Ld=FHs$yi^KXS~9P%VL9-oFuTaSeb1PA zHm#`;HZ=$ zKqiG<-rkY^VuC5rK~<~6!|lQ@&9UVA&kda^5WyD&RZJaO#El#tFKe93k?$KVX&4&>`Fu*>sBso;LWizwgNe(@a$xmMxru448B534j4w+lDQHTE@n@$ATeC8B8-xhje6# zF>hn}V5nV@bM7OQ!%dM=xMWj~s2^Xc?%I1<_h0Pa(R5zJiwWib{?h7MiAs&7MuX~% z%WDS;G_Z_h8;c!GOdiu2*?GI$W=Gt6yQ^!C3L%NK;#k0kl`V6bUhYZz`t|GEiBe$a zpKIW*|L^g+hR%bU8=ZCr{y(SDs>|r{I$=ciE;E?GBj6Wp%+J#NCC#RK{?K)~+fyx9 z)unZp_WzwUaGINBr$B;nzW;-cX1)HH5`MsB22xrt>6&QA!)?B9fd4T0l4O@MH2x$2GS7?5(ms*Zd4Y znm+u@j^*>Xql{G~6ScimbNOAS2nDW1^fOWdtdh!$N%_`djDhZ_j7joT*IxV_-@XA> zQ}IhS7OPDb+9Lw7@7sJh>C1^Xo4(4EPLdI4YMK4bM0RPA-3Bqf{QDMm+(A7o=LDzR%Eyem>El#6f+>B z7ggQYVaza7Y`9(DOHCVPojb;@T|K%2l+)_!YQvB5rlQm-2|jJmH}pHMe{So|++eEh zx56iAEs=USb^?ao$|kohButghXZq~qIrGcOJl}XdzIXg^+>GLAB9aY8j@9ir87aXgIh0z ztDpC<=TZr?1mh-01~rXiaiQ&@TUSTJ*hb3s`s1QNMmVDPn?->1i{Y`xMrCEK5 zew0P5)B<&=a5i*}y+vfGh3;FplX}aET$Yx8uGfoP#1)4l1;#tM&%cbom3+BMVBsk> zc#c6jy;LtbE1wZ6-vTcc;g~qk$hZ^QQE6$)*hX%6MPX;3Uj1VoSBLp*Za~;YIyU_O zpz{S;L#w>due#n|_Ad5@s&w`1S;NtWyU#Y`V*mV^heA0%8Ux`2?i<)#XywG8Q6CHZ zFB{ZF<={0e(*7}7kG*}#%PDe>&_zYAxs{H_lS@h&rtHW84t&_W)Kbu1FIt~c&Euky znv^=;IGec7mJ(jfA5vHU`NhckUKW?vX#uf1NkwFis@u=2npDPw5G!>OoDDOe( z@9!@!FYmM$$F=Lk9qH!2{W|9vg^ZN=+yTv6Z{cd@hRTB4oNcP%M;ON^Y< zRXia2Esizl_Xsf zrHFlk^GPw={qF+W!1kyEgV<|g(FScVMC2R}c)tbiE5fB(K7B+WvEJ3o(lw;Gt|S^x zD_rS_$ri_5`R_Y-imvOh-m~=^ywb0+IUJ&2>F_Si!d)+3mrSnTDmWU1urW$n{rd##%($a>3 zZ!*2mOV`{=`EBBw@S_&n42xk{Y8Gi~0HmAH0{$CI-doktrodE*8lqim2P)TnPkuKW zVoE?Wpz*<{WAs62c>a^_I!Az1d^ro&%^IF|jE!tabO=MXh4<@tDYQ&1I^5H>oXs7T zfbP=zn~E1(Q1gd%z;u|9b%|Hp0L6bI>|!PIc%m@5vOf4a)$b1;`jBy>ml-+VPCav!-}2x+*g2 zKF7p^A@d*k{FM;khqAPGfRP{HbguVk!WR(PCXJQjU?|&t1rbqzQ*qc!XzrD89XIn4 z9Z)7djCi%X$yfa}u^c$5wg*1rNBnUS(xu!Y)SnR8RIpIr&U$S*)Pa79n*QIq1Y4l5^!Af*Rfi zFVxL@8++kY{c=+hKJ&LO1r%Nwb7XSfGKQX3Pp;*(b}xGP1qaohTT}E6hy$M%Zy7$8 zJz1{L{1Gy=tzS0GLTj{K*4MxvKAvUxgToAnU%=ZLILLq6K0nTU2KSnOTfgkhS#Ljm zI(rWUs0LG2dkWkRe&3bqkZ&_0h>H{#|CwuVFB2OIO*B(Sp`0I$5etf&?eSpw@8{Ub z5gax|xFY94e@Gjg!PE%e5DGRqT*kW6>a=7_ zw`UyAVq)pekC8;-M3Wy_s%7)KJ4o+w5Tj}vstw(#CW~W8dY@SK?L0=O`FV?>k)l!_ zKkgA<6rPpeV;D&ZJhl9BYoS0DMa?u4*$EqeWuXqBPU@_WR|+_OP2K$GAow=TrjPi{ z@cA7qj;G++OzQYLWYEVs(&b~^ycrYct?P=BuN@rbpY*7fF7#N04PdK8*zB^kn$9{>;q2$hSLY2vH2IJ zKP*?Q%C{4%e1eV>*kJW&uI&=zRCBgWzdnef2Wr<50e*3Ot zSNrtK;I9jf>;qTtFP{mR8U7aRsfX9611f05*N>s=5ay&Fzu=IQz>z>d5Ij;09@^Kp zU8%t0JL0|>AuJ`}!LqiuP@i=kk=h!(v12R}Fxvi*=;8S2jb9MhbNrdaGVh(c!003R z@m`Q}@`hOOMsJtMPw+H`x!c|qI#NDd;mFpP`~^HnMJTFNd~tageFW=zqu_&@`Ta10 z--RJ&#{a@M^o(dHX+-Hqc~fwO(4!XIs8o zlcUuL@8d_QpI5G*zlCI>pGTMl!VbAt9xmB(|lFE$fmt^K*VG=?%*jKJ>=*QLED zCl=mdQG6E_vHEo5?dA{uGZ4wnEyagGBLV%5yimPJB{DMO-YfN1i^}&MEec}J3L)m; z;G@B(C;#;H`OD7@a_u@z3^+)OE}i+h6oCqbNy|LvcnG7Sb`shU1M4o8** z!`$N*YxK5T!T);hO2GdF4kXRf^LcJ^Fe2H+42uGWZ7qur#c6r5kYXYV!?4)~$%9KQZAaga+T$ zo?2rC6Ti3Ygux6oakiX);+8ocJG_f)RBQfW!l!34=DOqWGtDG>=3!8$7ez@(^)5#u z#DEI@LyLYaFG*MG{BEUN#NXL?dt1g@K#ZDdy^E4Mc8m&b9D{zPu*KKM;U*rM=CF6oQY#})@t&QVVN)UGu zuU&}$)eH!GKa*jlTNck}G|JnMqV)I?^3~Y5&`7hRPE>(H{N334jGW&)Yzv=JBK)lZ z_ZaXlB$A*S*Y6x9VdtqQh~xjf;P2}|oA>p@EAA}8K{sjbw>vW%G9pcJ#C{)^$HZ@3 znq^AsclI*TFV>a5LFuV@AN6#WYd6)q1V1FJXtkWEs)aL7v-pKtQ2mmVnkI>qzL+8j$J%`V$~?7mFcN$=dn^bFXyq>>fa*3~np~4ne&? zF5$J;11Gg|h)eK~FFn}Ap+fSIuhoYXbu^KV*2r^tm&I6i_M_6b_L#hwDWeppc^VAM zr?SVTO)Pld0X;hV6I@!wS(L2OPBP=)PFKR`Mk=I-$Wl-=g3{eCa@4}IRyFG>_|tC# zano&LqF(~)lH>RDmDwe#P_HxsK3|%3;~o#!xoiHQNB2lhOl1c5-hRn5LpiZ6W5W2N zeq0BOW{v| z=c!tTzdQVmtGBz%8-5-AM6n?+TyH zYH=~oh}x+Ym`)nJ2!%?xp3seAR*KQ&Awu+((A|rv zk4Toa*qL!0cLwgh-SWo)3p2t|ljXu%)$Og{M|^U}%9LxI0}O)NR^5mVErq8{?8~$> z*?S&`dYVdrj<4>2gB;e6F|C4~Z^m|rV}VwlfO!^xQjAd3LR2x>kN^#Z2AgNcl!yu2y)ne#rE+`kT7 zN&@#76X}dOFB2l}r~b7Qm7=yE41xMjD!GKnw^QiIw~y|(t9{k+W)qZ9_yNB{2rt}D zg$6z0KT;abYxiPof`mRULHRI7?~^$^nAS_S&xKeYd%i{o6=oun&oNJ3kB)=Ic;qb3 zv5o7N(#WtEnjGdP>-Mzh13(8+-hn2?Q35)FnKl9%U^&O}^ zzQf)&9n^j+EmA+ao^IHf@CS0AtV&jpU3 ziN1lDWmeZnmmj~-CdFp1`y1YZN)!9b`?_*cQ^6fNeL4Gb;Q}|H`-buvIC(>`D1{&3 z(nkabsRgyD0-@xK4;=vynfNbrrBUWG1D)IPyCwAue}W zmQ2sSuT;0T>s5WbJGH)kek~hbshzoDM)t$G$R4g`JyS98c>np+k49bXx?p^(A@JJl ze3{;)AF0>pO4JP`km62pp#D6cq+w%x)etZ=G60S|<03y0_O5hc<`Vc#RulH^AOm?G z;|cAmSJjVMC-73AnE3Vg*SAIFEBnEig^Gz@xas z#pbHGeHuOs3yYd?WX5*)~ zBcX!s*UPS|`g+iYqfMpjvSHegO)6e|_Yt=o;e?^D1<=Xt6j#hMf7&7v=g9oG`Ano3 z=k=Q0(*1v204AM!Wg6`t#$dL8YQ5tU!=Lt`iKt%tdVjsNKeG@3sGx5RWC-abXu-DO z2~PhKwTUomnFHw}Pe45tbH>Y81tV$%9S4Cur=AWg0UaS+S}xDKV#3SshO^Bhy!kyY z&KC@X838z1Xx4O;?jmhJN2HW~+6~7J`qNj2^pg-05~^!yH8a>gYuHju%H>5!So&XJ zTClC2Q7Oij)KrSPuJgg3A}~x%^_h17>37UZ`E6SN1BtGlSC-7t05Xd2frj_EKJPsQ zgkDL;WZcbPFaa#F=6NwXkZL*EmvfIU#o#ga>|1e-VJI(oirax*r|*i+>o3092_vqw z|3ND~AZ;QB`|Ax-)YdRpqo41m`Y3Z@$SE6GBJbUSMcJ`BuU#ibOEnKCcD4Kimr~V2 z$)wNy40?|sluL$E!P5Uiv9YGCK1b{XhJ+iAOXG zn60Pt_!`{#|L;S#RRTaRGI;m+`2YDTE>eUuZ{9L55nI+dolG>Dj5A0LXX5{D23FaS zb15m4q)j0!DjO^srdc6YqKF1BiBhhqa)wYj1jD*hd;u&=eZJx?O+sx>3djtFe$P}d zoU2rMtD~mR5|^e(RPGg+zvZP&k%18T1&CIWBIeHS0U=JyHBRH6lrjiZqVp5yz~XBe zkl+yoE=m@qTd~tRvKuut2u>Fg-#&G)XD>!Zq*%yttlSr5cKbX2b6r@K3{{DQPW_m} z1|ymxIgtb_hMY>omZe045*3}bnAPMaz?!AF_g~%!g(;aLL%OhN*rvH~T_Q@cvuW@b zS;QB)@sYGkCFX_#h*Ua6pq^c_>!k#MYeQ@eKzOnI79cc{!Cde{FT;OsxxzUVIU>ix zP{fW;&0MOQS#%j7*$oDf2K62WZ;@O%GnhypyXjzhi0;rMlML&3X_$SA$Fsa7rnLnl zo06ukmrCBZBDb&t^A#XFD(FN`WAkh-POWxI>b^kzi&ZF;n<5|n=g~n18=Gre;!B|@ zn~pdY>);07uR^R+hL?AEBU>rHNKAr*IoFNJHE#}{(2Siy%s^Yt@Kmo80le0~n} zl#A!A00RaqG*}GkcoJ@36^o;y@lkQ-G8umokdC z^hG&B9r>fjy$+)Nty(gg-2uvqPWJ<4zuX2f{@f(LRxw#^-hR4IUQ zkghPVv`b9ibe4JZjt&y)ZOIabRW*Mr3UU_^q`SzQ-v&Xksbp_bet zm%Nr*B!_)jBdms>U&Pup?=m-GA?A_i^LFi@nwF-bp#du_Cx_qK+KLlL z^{2Lql$rUl`dsS2>6D47_m~+n90OWIj^Imd8(|q&_4ipAcP7NM_WB_#{N2$YGq<1Y zqinP|7x<#ephUA+!!&6F4b|XlQes(GtgJs{>4m@TKus?k?Lwy4<GoR zf7EiT!A&yBl@@own#Qr4N}FIp*ruIkbntSZVZOq*pAQLcD%d*W8_s+&qi<{a%P(oR zr^9bTW7L*87fzL1d4h_0&Y-fe!SV|*g2+e}n{mx`0oSpx(^ZWZI= zMcAq1Z!WiroQMb!K)AvGRy%S*L`9UKrl}ddMO@h2978FG1+Xq>_?wBp&(Es@Aouoc z(ID`s*w`vH`8}&rQTYLz!~ZH*z@M{P1pbsFK{3AG9R~afz@1*)zM`WGA`$UV-WrVL}fR(tRV!?Mr266#ob z$NV=Kb@{gkBjBdFNq-hNJdup~0|lc+eDHF(j|jvHn!YdP#IQ!~TOHo(Z0g6rS9)h=HiINixF(4I*DFu$@AI{{4# z&(+wm2p9QthG565f>_T3ECyR?^Hwy(p1{s3BG^V#6Winu47S15&5FyqsvS0ALrd~* z$7A`MLaJUHj7BNq@-wL-9VgVFI=NBJ?`((u$~m*_>e`w~UOUP#9J|st~la zqAwArIs?^dq2UqN<_WuwL=^TX{Pp_FkFKXDKB+NSE<26EG-&g5DmV)4e(+2w?-#!3 zjXiItTAvd=kDFo|8k9RunOkp%s6hOnf}%@WSR3q77ucJ=Ol)PjLFH9D@_WS-7suJfO#4Rw4TMp(2Hr%5TK&Osqmiw zX4{+!axqP#jp+U&7X?a^45#Ymdx>N%&bwJmAiMG4pQf z4sgbxOo!3xswzqfZ(mQVL}yg&xVoFQr4^BJbePO5Eo{1K+t|=xci)A-K^TZ})EqYZ z+kW%O!-^Z1Vque)R~hbXmCRq;imI3oU->gAy*OT+&4#uq*r{)w$;%64nch?`Iu^++ z&Q=#{Y6~;*!kLP*wSu3cg`V#;_h(LNI+>Ayoqn#8+^y4ysgHNcqHjw(=ZukyQNTx8 zZgo=6w4CO$>|jqAV%tZk)}8K5+X7!r@XTmx~$w5w*3 zZrf!QK6m)%&f!SU)PZ#3tQ9>ulgH|zfHF@4b;<(yz0GkeeASK$1MX|#@)^*m78f-( z9gqTE%Dfx<{^SJl|Ax?&0EHd5e`nBAjn>?jkg%E>JHXO;^nO@9@_sq7T{tAB?Nva+&89pf)X+y)LFx2R8#271eA7 zpH4)JS&``WFl`D#BaVb>0&>Y%jKhKwa#S(t-zvv0GIE@zrK!c;DdM|wbCsM{_da*L zgx0!{K_*;Z$57&f{Wx?PbyXrnY9z%&RUkT>|i= z(UBq=ej0ES{N5QI+8G7^S&9}|<}Xe*2j9Q|2^@XL-U6%jgJDfL$k< zfPj6q3X5bwz$T?LJD$SI1+8?lxR@BZVFPx-3|EQBSq5=nieZE%qw%cu9FAP3)>iy_ z!vOJEc?%1qgn4xmN!YbSPi(HBjF7M-Z5pIUG`Rg8Qq`Drdx|x5Gf%<5xShPs!v1q6 z3Sh#4a39Oce<@zYu3`+27VUlFhzk;LfRe+4Z%(sHwQ++ISgY}}kB;buY2+u!Iviso z8D?hed-l#qJ3>=EG(`Prtm~6Zw!_M$V9EPWvRFDBgMv16#Gb-a1y6GCIhh!eO{cpQ zC51{ps9!=H1vL{Ww|22cl)>7nVN~+6VhNxgcJPB8Y%=KNl0?0I<9S6xx~H$oRX&izKP$fl|A&bn9| z8irho-l%Y{xZ@i9(61){d^I5n3tP9*&aAWiQI<8WNez#kZkl>zk=PRlgrDl_tuv{A zkSDR2neEwacB4iNI-3ZT%tfx+b^b5}yu1!`9C!co2q7oW8yEl=R}rntXCML0>Hy^N zziAQBi&+YOTtj=^&GpUB%>m1Lv!iLkTJ6U3XlGT)5|zIWkMxqdRq91(5I@#b+>wiF z)`XLICWVy~Br1jzv&HJajXym>n2?GZ4=DLDP4!kT8aLy^#>Q{M8jftlFrkuTM3cav zU0Wq8{|Z;E5Q%{{f=MLv8gV~f!UK>TRli^>0+gOuZ#EV~eQRZ#F$TlQ=u^Ahkhm6a z!-MUJ7ap$s;dlc*ZPi@Cds$r)JVdlOS(5lY+^JucG341BePte*OVymd(a<>dC(I@Mr}QeIbpkHP#;Z3wVkTal!D7dOjvcDCAz8Dud#{KB zJm;@pu?4vTd%>Ay(E7K{A*Jgy0iGP7g5^MC#XIjQ_rr^BR6OEzW!6vhDmgCv>^)EvdA_l-Q73=rgf2M0#W z79dCPdHeWcc*V`Sb$x-?Sc#3g`?t>e%V}zORlo{4e40R4PUv>`++x7W;zk8Y@aWJ z?FX92g?^maSDF?a1=QbP8~g=S=nzXmSdAWM3g??WF^H6!>enz}1qnZ4RIX??xwvFp zb8rR)7DrH?Yin}xCUB7Fmo{Q%+F}uh)2lE`e&TiEDk3Lom*G(nNq(-n(CB`aF z>z8us{isQ%V5X%*`yHz~KWykT!YuTOL8AeO0e8}5Z{F1Be`Lwv#{25b`-+Q$GXyNk zQDAycJwign2!sqLwEA8w0OnhNl^UWrSrYsNJ3Biq6Vrext68J=4{fT6syQowpbG&I z^cR$be20H50Vx3W^)X$ru(6LV3$wFP$SnJsO>QPcP$AJ8Ifu@EpN6_AKJ&7NjSPd~ zJeH6tGkEVDZtjWgR)ST{qop-UVU*Mf-lo_i$U;)~{6V?}WTVxm#rIgoNa~VP30ag0Hz_P`bJr zp`aPn1ec*4+lHu!=a=H&r}X?_V!H$P?_nbIp+D3Sl1PJzi)UldZL~a4QME5K+1ktf zLrbgOmW0;H<7V^Sd{TIfmvOUKE#oae#;yO$$;JiF9&2$8*RCdw425A&(3h07m)KZ& zqCh`lfPfvTwxv2cI$GGiJhs`Rl$f3l^iJS`2)_I5wEcCQ~T zJT_Bd+XF>hT%0z8rsmrxASF#Cd^Fj9c8til&3W9M%!!rB#DyRWq4)xolqycAh9M(E zrLgX9XuniHQ=hD5d8Xg4(YwAuzpT?*UE?xl>3h&hxk_dsOGcSY0t*TiQAB6q8R23c z3E1!tl>dbQ5=Q%)$WHO1CV|vx6k$|h^cm3i5hlPVLy_G5e5ePQlE}%iBSqxdueUKd z1dhb>rKduK4DJ+atMm9habidmwY7z7Gfe;bMS_WWpx^11b+O(Kuo#1;Kn8EYO^dC| z>*kHBql9~{duYg)HUo!@Vi$d}$CZN)XxgzJ63C1Z1KsyG@NTQ(H1^xQKIo1_7vGDs zoa7J>mwLu{;00>Ti+i^V{0@8WOJQ*Ml#yD1ngY{qsg%$>&WN<;?Bx`o+dwuggH3cD2IC zdA&P@P*~kB{gO*{)ZLs|VkoeA{KOixP(C zJWAFQwD`?vyHX{$8(U7jLJEc%CvqTfuZRqKY(>`0HufcVq?!o@e|uWO45nnvm&|S& z#{8|Z2hzy{D&8%D2@1UUCa8zY>R?w6Lw zj^NFkafy76MQSza10O5{gGgZb?`AE3tO|0M%T}-hc0mhE?qOY*Ykj9|1CM>5{_iug0!%t8ZNrHnB25&+8wc~3J$U?k|Za|4YRBg91SZ6SA@Qp2Cmw8fDXP=f^d$4 z9FA93P)Y?GRRT;VeM|v))|pA<`Uk$)}q)8hJ!IH;<<&(PEusu?SLn~<~$`26#;CAw|D;%a_t2E=@_+Em3 zRu&tnm14n#RRRT*`+J8^3|uk=AhhV)rZ;*%L&o?p$Yb|6br(~dly{mZ?3ii5x|H=SRN_TCO zij$?V6{NGHi>r@eZr8V-j)d~Yc7HVVsLbr&A(Zv!(6@B2vfLpesA=il_rn`TR9Dl; zI!_^5!9#|*QMsyZ{J_HM&1`6F@3XS2f8Ux4CC!*HV;iIIG*Nh;qs$fh>od4M0cwu9 zGD(s74QfuBn9>l#k^rk&3LoQm>|Jor6D|7Akn_F6^L*e2Fx@Ps!GMGNmgBxe1b;9# zCS&M-uko>s`~ie*e5c9zFYw7m1dRDS-@rsF5HgtlNt7Z$Vf% z)j}LslVGQMSxY#1N_x4pL3}J!Lw6;3sA*jyl~u34o`iYR6Pzl>emDS9Wx%e{G^L)Y z{qD`kz-Xw&D}k|NYrh6}0r`bczvP325PX z^(P4pj+`^q zw%cRXtfTer!}1u=hZo4`7iXlcnSStifvMc4%>WV7uZQXovJ;+_CrF2vtIg$AH7jXi zV4uVShl8m15@ryzNdDS(NDug9Oe@mky1=w3PDIH-Xl z68@bog&@|TOk$NL>#0?DSje40GrO&$U2C>Bu}X~rn?Qx^lu*aV#a$1;GuL`@1ON9M z|M&4A6lq3!`pv8ua-D9)?Eice((hzuF4I}%mgbgn<`~KXtcU9xHbDo2vmEc`gN0L` z=Z$~t8CsmXvRFo&q8S?Oeloxege&<@L>B{^ht2k#;QzPlkexvljf?Vm4b7PQ@&yYn ze5mGCKee-(>SzjD;%dI>rO0G)6%9u^1bEon*0#Q8(2>tmSBEXB@wXgJbF_KlSt~Dc z6!a*F3}z!k;5`5Dki!v0Ybz_a&x}5SsE)z^Jz`@PJCxo1eToR=20?=6|NZ@bc;AGn)>#L8K5t=~%b3OW<$juW*2HWh{|L_q2mj~*kb9!h9;=~OPaO$MN z5dFi^WT{B@`ktAii{jw&i}U|ouy$&1bMw%neBb!1z|U9Ut+FVZ??x3;9896-{PQ9c zLYO%*Q=y2&)S3H-o;;8$#4F3B!24CP@0NG4u7TWu5}SBu@r?RJ!D6nK^1RnoNYBR2 z@NTUt^BZlaCwoGNP0?uKN(JNys+o+fMR3C22l4QN;GTl!29gFC_~4tAO6Ui?5y zUG(oST$3aI-WwpKT>r={H-v?*QrTHBgEkWe5F{DkeO92o876nGz-%m!5KiYAq(}=(= zMG6z;f7%X_1D%+g?df;UDpJ4)AA=S)URQTRq#tSH4I+zkBnWyH{x){yqjkYD7L?x3 zt4#M&Uypvipai+|wA|3Zk_Wy_8n_$o+;Ww?u|H0Q)J zn3ge<1du$XT3HY2zPKa`7$Mbr(jeS|Kp<(6Bp0v$;{s5n07k-qA+RzU>>_-{SiVrq z-mjh&k1Mg3zu8Gy4I7Sn>$HM^DgLHYhRcl5Cq0qxxk|3$5hL<)U2;C|?#&5?Lcvy} z51I1voce{BqY{1fu`QfT^u$d2&hsBJj@r5-!9i&4h1QP#vrieWzrIjDvCGU&Az*KTERU*6J^T2*B|oi7kEb7VG=jzxep!nhit&wwREl_Cd<;ug#K`P#4W zMc$)V%(ENjk50UyY~w4uQK3BH-i4OpEDzSUa9+-8k;!hVvW1$1OrZm zmCE{hsx(33~VO-FUD7! z0;5fnVer_1*FOgr7a)s95|)ypjuKBsr&eqNY^A#mku9flDf1Koy|$IZX6&%(&SKTV z(BAd%J0@J1#ky-puA7@1*z9~$R#KA6?~xH8)1j!tn_crP&Yy?Yy!cL^f5Z!Jpl%97 z(V)~)mDFWZq=qM0-r!wqtO#S;IP>t3<;!YSfTA0s@XsK7M9h@~vZOh~&U*R8ve8kqBv zHt)yxpVO`En)=V{1K9F)m1JJ$KPTD3>hhSTX-FYS@UG-BFZdzu25in3uTCCk{{i`=p98`D<}69exq!>tO(nl zn-_~5i}$sAz)gygYprW;H&m4sb;Rer<=Z~B<2?3Qt*5aN6yC`RSbYAuBn|A|(g?T9 zIe2zgcDZWa-ql?)xcs!L%^60@rZ$Lm{Nkv#{f+S<+@U= zL6JYVWMOrF=Net^DFb%-?av9WIzfANqfXK>^JH-{FyYq@*Vx`0JvPg=M9)i;*-Nr) zSp&}va;eS2+FAsephD+_M7))G};PY;-; zs~H%CO&pLBAOYsmjy?BSoSaKQH<*=`70~%}BvZ^SE=G>*8UA&&wuY%$5>-&(1XR@v z6-p&E5~pWpQIaKY9v-X$0)P!;NmW&nG0S(FXuz%xE@H6pXAj^F@9D`USbz<@^#H}S zfRGyLK!Smf_c`-iLo)6l(zx-pC=fzGyhrq@PXmwfZs|+ec$yBWJc{t z1^I>|QGPl080rdB^K1?(vx#wUL*2+fj?^?h9V9r1kz znKq72FcTm0Z9OA*KkYhdZe-}|=#1?p2n9^p!DISf7TaY^-)-xxF~z%7>Q_vrAMB2m ztYj0J2Hq)jO61K=FCb*=R*?)Evnuf84IVt0t_mY}CekKi!dLP_=v^Hu7ln+e6Ct7_ zIg%H6-w}-~0M$I? zjcL2BXIfoVHFd>SGbz06a*}oVzSlP1WMwq{${JRwTheo@37#%TSvv8$#t8&R=s=<-{c?Cu=OpIEk_-Q-+zbailpL;_>40vYMLO6flJY{uvYDIgh}y zYzj0!E}LW#b#emZ`*>Ln3@DxSC`mJ5Mh9YuFfmO4x-AfB_wfOSM9BMdC7v`*cE_ay zD1Rl>Eqr`^0c8$Jp5ns%yvyLxJi}Qg)yP2)jK=p1;jY< zU6ZyY2-MpWS9&b;ScUrqK7sGBgpA!`3T1+_znM^~xJQ8^i{$&Dj>N?S6x}4LZbE@M zRRam7(yMjZ*EtN1yo70RH7~NaJ=F<@xo7=@q<|9ZK ze74LujzC^V_llkN3K&fBXe$TQ8)>B2N-DEG6Kjk6dwcqqnk&8Uf~Q|Wswtr0R=$q2 zqKl^%mCcu>b#5a6@8gL1Je)X!0M5a}$th99qD2pQk^eUveZ0Rqbs2n#jf(>$v)*`k zr~l!&pyvNBd_x2DFNb#0XtL~YT8yd-9MicxgY^b{&+{2XJ3yo2NaAk=TEoD1_khpGz`!v(FqLFf${)IR+Gy6%Qvo;Fn*(=oTSZGr zyYrEdF>1w?(b7qZit<>IrRWqhSwF6Xq*p%X@<#E=vYoo@5E3R)CLmyW%*|y)GnMY) zfj5M_>aU2@b?$IBE`pg@h_3myHm|197F4%a*7h;%RN2Et+t3%Tj zT(7(G3~rC8>uVr;#cIHkC4Ou_$;xNDGkj6 zd|zHT0DVmy@%H0uOH#2XS+N7;i%x#}U_T_{1rxWm9)A1*)MG zSj*$|)XY)mujZ@nMTukHN4QIA6+2y3(~IXcAC&_RHv|N5w0oDeu_!O6C8IC(=xA-8 zmR*;)4@@+M7dsi=o8>Jmd6wS@ge!A)t0D1!m9DslT%F-xcD^I!zjh92Ey^U@to6qK z7|J(3zD)1da6o(XoE$&i%z%(ms{mJ6wkYol3x8~3+l16XI8RtS(_cMwQP4-^AERGl z=dX+@EDgCYyl%M_~%}` zaoj9Pni4e&ZH;SyUsXhl%g7riI=r(+gx^>6RIolP@>sDOpwJ8|Ep4gV0m@sM;_Fwh zvkmUYdHl>tIxep9Ldj$dTwK6pBo;8%rO~lIEZ2Ew06cw^m6Mg3%l`Z^JYD!%Fl*MX zJ*6odQX2uF%I#d^T_2|oFmQ0UejqMu1Sk*`!_6i5>%v@h5miU!;wG<_mxsC!N?|aQcxkgMj3;lqG@RKczWss$22wsB zHeq}?uc||a4wFOf5L%>52eb^(MWfn7jXHn&FIyS|%|vXTH%yiCAL>bFz{)v~oII=- zSZOb}I(>I?N(Y>em@{M`wr$a)hZuFp7^JnzRbuun+n}S%AWP@L(dn z#j!N=;zAqj3t*-#K^mW1aZcQ#8C6zBi46y#(O6Gy-PO{9;Wo0!Q@89#JqA#~J%kAB zlvGq0fRRa77T)`lyZ!xvb8g-$uecZufMm=`3FtTYSNHAe6N2M7zSp+!(Qgh+JIGut z*Ft%Sr+C_MMukyg8^@mYhQ$RBt~*Bo;YvBGesSgx%IT(@tKnJk*GesF_k9Jrc?C05 zSEbe)ZdM>87dd%u-xJf<>$TVhEc{7r!GUrifHq@me0rC>!ey1D zIBzN0g|V_Ep#uu!z9YwEM-3B+s21u)L&hTpz5Dk&_G!s;(?8?V08e|T8iMcp;p(I& z`sBi}Bl<&!_xj8~QgKrCw|8p>aPW^BK`56^G3Tv^%%1BLG0pZA)uCGh*3E=fYYi1C zRkD%eb7E4)b)i}bkPTR3`J32mQ;q(}P(7ETDEy5~QMY;z-0WBqO0;AWVafzu4w$+m zNsfq}q#P!N|LvPhbE;bc#RSDpgPc+f3p58IZEk}?Tu7FF5J z(|Z7ujuis~XwG?Q27i*H9zs-c=}BHZKLJWiqF9QYPzE?1D4uW$OAZMM`5r?cvPoZ_P>S@fWz(wZ z1WFyZ%R{g*CK6Hd;0tHrKF!St$!ub0Bt|xE>w&ApSWStU{xqB0a?~oZz8!7GCnSAg z$cXXKsZ;Jab!64Su16goO(H(#RL)mYn-GO`8AqPZ+7|AV@bRJKe|UUH{~dW~;0;80 z_)QKnNn&A6c!+tNsl@Le$G=ZIc&psH_DH_cggz-*c1owG#@%eyTC4DmD3son*-ZvB z^Uw=7Q`cAOHP6h-D)tpIS5=i!2KxC;G&WW?-v)jKE+W$B%6FUZfnU2}9|xQ888{z# z0FcwLjvWwvga!38`nZO5Z(gY1Iupf0)>7|Yy8}bF_87g zudEC}5`ZbbuD(7})2c8vs(~|$h>;OE2?^ErnSkPK?_V|mc*`IIAf-zs3}n%+RAp zK#7p*^@t{lKd}u1?QS67%BQ0b2!-JRJ6*sT-d9c;?M+xSd^EnlzaPUy!%9*Jh1Dst zDGJDR!N36EXI;8AZ$s(X;X~eL9so2|^U0eCYRS7vvON8p3)*DJnJvRm_!ELc>r(fB z(AR32{^ezLz`&{If8f;rK*(4Dn##%p@v=N_>q!&;*%t&aJ71qJp=!q3@*x7*C!o4& zu^v=b)T@#gX@oFY}M(@Kin8t+PA;mjCsDUZfajv{|D&x+%c+Key)*U zX!fRQV>7>S4X`q?E@kSn)@deYB*l?-U;d;Tx%-b zp1oU`u#s@zzJ87C?BY{CDYT#O@dKv#`2WGh?eNo=k99Vio%e9T{|6T@;rGYJA_5jW z%m2T*4T0jg#JWs%Kx4h0y|Qu?b#ANKoM@<(0exhVZ!#hV#kqhW*e~7-~u11 zm)MxyA@KP|nEt>kWO|}3bEl&ivc9dcvTohCJ{1IiE{IRxOQ20cI4jzL%6pJ9^zz9@jjry!84-fk*+mp+<&HK zn4O&*iO1;&DPJAR^5B(Byk4P0UkLnn(;&-o)fgrzGtd;3CCJCVMuvaLSVz3J|kT3hrp!7JkxC6noF_1f;@TG7VY zISIIJhpDzseEbErQkgnN+;Dk` zJAIQQU#l33`^o8|0q!~4m%ZNkGjpqqU-Pghi7yHWxb7HzSxH>69*14_G9X)Po(9Q)D8kITjG7gF*BkqeFm@ZNo-ZWL_d7p` zMh+Mj2qNH1`Dt@3(Hr-iU7I5QcUWY{&=Gk93C-L+EVq;5%FV8zdl_9MO6y%~iZo<` zvUyCh`bhj`1w!0_L`_?!TF9`~AncEFfIg~4NnG{NH;JI>Woqpzh+yGwxnlZE64R$W zL};n#CuC`$Hb&gaJge^6v9bj~z;=xqF@T2s2tVS^+>7oAcN@Z`cERPVukhC1y#@mq zLrzQJ4&b32klBH8jiv z>6if55d=u^6^#=DUJdWx0O^>D3Of+GhK32iSmWYC>*Uk`h^<~;UPktBn%8LnEjfLv z)ZN2FoQw(DqbfCOvelGv9X$sJG#`*M6e|{B$`mxVUr<}S6V&EFb1=KO7OiqKqAnsj zBu<53ZQ7F+8ZTB5av-I&z&8Bv*U=*OFv$@j2X5 zoR5b``0=Ut{?;H?DJ~XDWXOQr)oe*6RC){<1IvIRZP4&F!BhSthPWldlBLIzMYwb} zlw8BM?XXLxwL^)6ln?a4<~Hz97TKts2SSBk_X75pZoNg^%TJFORj)$Slw%CygP(mZ zd++5$c^pPyI8O;3VX#5zcF*>lpI7jo!}-zPi5(s;w`&fcBGjk=x(VPh-9Y898ZlOl zN;s~prA9NQG$g#?eGZ9JCzB?iGUpm_&BwiO@3%pvMLeLc5GDYu+#K!G!fwnp7hutU zpHQ+(59;0PCE>5QBrR%lO1#$RxLG@y>vxcuu4!-QmD023`Fe}~dB zBtxJ1Jx-gDkj4w=UOXd=b42WP9ZdEaVICBuL|kkt`m#V6m8dL3+FyLuzL6ev zkeg6?5i-#cT2y!nDHk|8)UntQm>KrWbPuRMtL#sPfTRzo>_9{yzNK(~;6BF>nTpS&Le%}msI)7Ii7COGX&}Au~`hft6 z8Ct`$!YMLL`4V?XFf=@ZT<;|M_l`OrXB}q9eyq0>QIR=(drHc|o{mEYlF+h0$D5q0 zYOyyL5>gP5V^V*pIGD!V>}tp zim1&cx|=jURw{mN7Q`DpRVgH$`wMJTJG*b)+|WF6@Tin5MA5=GF3A~!K&gZWWKq}m zHT10WL~3uwajDr@8WTm`2BB~f&jSCCsG3TLcCS516VBU&Mx`yfG!+b(@PDL! zOw5Bx&ch&DbUZv0bMqkHkKNBaU2+N_oUE+KTmxj-+kCut95UIgon;fB z9(eC7!LR{BE34us;Pr5g0ogy`RY9Rv;!=c^>&JY3QUYQ+PLYXW(mG9I%#grks>^WD zrC~V5P{|)gPR~G!L(&Vlm5^hO*2;VbtC2SjsuC2EA6gKiW*cKqkeh|Zl*2rl7h3;H6OiiZGJ@D%%~iM*1z^i5k~k-v^Evu}Q2M(fW708; zkRg3tmR;vI)j!E2h6rH40jg-883*8J2mq}1EnM&K|Ey10V~Byo2bq~;7+BbQASt4( zyj%_?kV>_Zj{972k-2u6T3K1SU*8TWv}>thfXt*NOBMja1B{A*j=;Zm7%_bj-RfEU*uA{0~n7?94yeeYYTdi;L6fYOmD=V3S7`7(99~UNYi8# zbri$Ah&DIk!su8cYCk0{DK>orIq?)&cv{}vk2&0BOzb@aHl`tu<_+gjrY}$`2aSJG zp_N6GsvR^6*vCwicwhJ;s0kWsKA(>)BI{liEBS{jx|6GzA0X6jV3WtSHC3v zWc?9=TE_m)lDA==^Exjclc#TWR-4DCIK1%KW%(?p)(-2pR6ygbpk>+3HnL+}{Cmg` zWmi?=2PcJiuW|RXrH$51hUo78W+)xEpURPXb5H2AIR3;BEE}j+d@xO{UOccJz;$B; zFbOTKb^VGYHEJ23Pmng}^PGxv6ku48C|s^B1P5tajU*N?Inm@QETg<^zoh}@uYf+I z_N!i-=M&W~Ar}{T)k;TT8tdEWXandUYRpnMfP3fvq_bbRbhQ~ah76qd?0i$SOG>RY zt8H5+*6~2w58pH{<`mroM~>4nF}4f{Ek*n|ZqTY@?VrLxoh^@Y<+|p1U#@ci#9ab%y1z3s zwS<2hwiAInUx;v|#o^HHX2iz!WX?>AG*y#EmM42?-=2XJtFWM7mLroN1^mqUHS9{q zgZCmhVF~lUT!7meiPF{pAVxZ@H>~x@I3DUZVGFkIrWgqkUbp6?39eT{>G!JZ@-gFw zb+3>cS@Xin!kz{DI~Q1kpUWc#5IFUC>}^l8qldi_vwHjp%G7y$s36~s8eW%Px;={o z$LQxc8PSAKa_YX6&v^&zc=4Y+e1j5``Sl_bPhOT{48g{Hda$8&zhnfGsL2&>T&K^P zZujcnyiQ0G2scCNDo!_NQ&cw4Gzw8pC`TNdJd7Id3l&q{vIZu=&!m`^nyObDD6%Fp zq10Erc6SHk^XT%~y|kpM=p#J~(89QG$7iv*TvPOu7XcO+v=d6iOr_P_$z-l@_V$yIg*o@_ki}2^!rO=Cp7A(((>AyZ`gd zR-7+DvdU5J_Nj8fh@peNJzX15jsqX=F%mLSO5Asw`j8iB=Ibrvj7uvn~SOfby-Q-=++iO zLj!X{LW1p^D;91vMAv5Uy0wX+Aqs%>tXl(MKM-VWbJJYpvHN4fU{(wT5Yt_vXiOoz_ZFy(0usrn4wU;Dm;pnzIhm7jNUdQsT~q0 z=2}|rLRf~OWE4^P8-KOjAFEQjH%}nrBA1>aEL)-r{22MyvB$uDnFUz62onRJM|7Ym zra)eNmzzv6_H`kM;2UMsW^3@y22Ch zLB1yMWG$ptF)f3!41&DmQfYtl+O&8QJ`p#L)Gte=xg4*m%k&(sig6=al-g5!$+?c+-ias;P8RKTy00I7w zx2disw#*6R&;k8`zU=|Gii(Q2eVq?HkbiB}>u1lil?H!gvK80uz)dH>ayncb+iJ0L}UP?DVuW;EMdpDBWWym|H%ogJ7J^q+rsPNS-ecyMF9QZ4FdRgyIZ-_;T`yr06ifL2cXCfxtq|%9i_|sEokh zXO%LtCA?*?_I28r-u;GndYdiN$+Q00+X*am^@w3rcfT=498pb(f`M91s-zJl@(GPbz?3JdgefQt^?6AC&yLvwRX zfV;rinj1vY^TiZ6fA_H?0%U@2(< zt)VfN{c)UKd3^<&P23t+l)notYyzus@+OIyN5b}4$LNaN7A?(+T|C|-EI(+*iDB}Z zSrL9;TCpaK)ZpOwk(E4{Vx+gfiKXUVl^-&p>CY}7so z4~p94dri{w6j!|!|3<>EY07Q&zr59d7$%55J_2A$Ab5%c_a7p! ze@`|4aDIM1Fy4iGUV3c4ZWY4*=Qx*=2tpY$Lure$g>{#!brVB~-_j?X74_tl69xEt zj5Rc-_?`v6fNdA`R?8_GAFJ`O)^{p^HJ1YPUZ}d|`n*>84^Vg76-Xh#O%HJOS^P(r z{SRiU=Z`=l{;T2B_P;xNY(3(Ni;4hsHl)kC6B-D3gKt^6gTddbP+#(p6k_B&UB@cZ z|DK2E%rFr1%JuQI3lc#r`ZP)9Gh9>iD29q;v6_SLAZkjsx%%04ZTpT^EjH+S^4|~b zBMU_bw89<_)F7Xa*Pp%Hzmpd-_(8}|gk@y2_PTICyU?Yh*2-)#VS+47TLVTtlqNCFXx|TThAM0sUA*2B_}k5ymFVzGM@c*x@SfI=iv`py!EC*R1$>JKM#p zzW`r%k~C50^RR3G?TygIb05A0=gGs{>Epy4RhHtB_kqbprJ};=qgXKP0nK~NM#e?& zyHAwOdz;sj3Fp(atd7X)9k}=9cj>jOTy0r@@QAkG zAMHITxr+#L0?*T%e7PKHA`I~jTtueA=+#0=%=YBG3B9r5s)Z8^e_@R3^G0z}cJVfm z_l=NqY`w{lHLd9SZJ_ILI8g^_O4v4Ky*}aMAo}8&7jfVeZ*OT?=eo(J|M2n?1n>U1 znOUK}>#sj&R6-_+IyXw9vKs393B*O_9r;np4a5a zzC?0^Xr1?6e_K17mIv*mJsYV%q;$3~KTdQozCco{@_)BR|FYmNh8;WG?4|jGS4*uX6|pnW8VO`&B}6V)Sc4FTCH(vfK{L;Fy#Cx*q*|NmWm&Lh0<4WaG0dS^sTu z!D*pbgiEHTGmw_&o+&ZV=U|;`xBh+Xwm|aXCXtg$eRi2XxcQdqQO-y?G#ZAHjn3a* za6-2h1}9zrbus%ORTQ4#?Lb`yPFO|Vx?)u)UC1(}E;yQjht5Cm)DicUS(Os+Ak?43 z=W|8adLG%%EH5&7#Yd!}Dwg+s9DYgB3vUPl-l6_8@lo9(q|e78<>#YUR7D~#k7sx; zx|Pps6*dpA-R&2h_^*n|9_UeqJU86JFDp;`pRdpc&+|T0WJO0^Y(fWuTEi$EqMV^W z7Lnp8v_IU>nw$mqus1~jHNqCU>;nsQ#WVt82?pXk@F-M9^t-Vxf0z=~HzCB8~DqayVUZ+pQO&Fsc(nlE&#oyd4ALXO8D-U!o+|%mFES|LY%HOtfKSzD=&bMUmKZAK`Ee?Hro+UFV zKSfvIxQ33FlAk_#ULK!=^U;pDAK>bY8n`?!WhAKitR~)~JBE$Xwofq4GjHoMnAt$< zV4Asx3x6o71}#!-;gLkn&C!AG$~(qKmc5AVJ-tfX7<*WN8x|kaS_y|r&#V>`o{#5@ z8qeLqfC>`z*XmXQ=_X>NEKHF*F{wD%g11R>+f57{3^HFBw^u|2WaN-B68;L*WJ!27 z$9t+X5@5wdPqWJx!_}vrf6$Oj|S+1+CN| z&**hT^hkSRh4x^l8rM|FciJtz34NkYOVSl21Anj3OTQ{Lo~5*DFJ|ZfXCKFLaj?>I z%g!|ZWQjS=;lk_ovzzjjHC%W{F&`#cgg#KxZb(T}TCy_N`~LAO&oAeB{9+SyZRby~ zr;U}c7O;FTFClw51x4XI0nW^I`ac~S2!EgoY)@-n?Ur3~2Cg(HDk^tAO)lH}b~#1^ zKUX(Y%WKWcH;g1}*F3^C&X|U3G9|3P5+LAf;^%78T8#Ir_{imcm%+H-emi-V;WpLIh>TH`SC^N-@>+TW z;*AG-4x8}nRZfT`7S{;u6T^$V9yN|}yabF>I`Q6-9WUN&2O?rWKFg5w0T#tW_X3Fkp}1H2O74Zh?;=TPdEwf9c*D73F6|&J~eAR zk2GU%@LpAg?-7dPVar&0aD;SE0d9L|kbW$B69x+G+Af)l@i{!qi3G?=v}QpF0TQ523MjCwhUwxdA3 z8LKz+gl3ix_&2aIC&puG#l@A(t1SKVKLj z4qQ9yh{v;##As_g&+EjiCqOh6@F!obi##F4;y)<4t@?SKLcET0wkGTG^cB@S&9$u~ z^*{lfdDaBE>^!I4mA7I2y?&MR`9m?D+~ZW**&+ulD{c?S(-Hd{u0~$hShPtB`k$5lbm3{P7V-eU%tDxq1Z^uA?x2XE+&my|vhUWMoYh0RREhOUXw zf?_xP&fg8RUcTF$yvD*}KQ(UMQy86~{S#l8UDkvZ+oJZPH$oq`R_kS4(;#Z^?tXNQ zw!u1#C@JlPXZ3ZYdRbP68oS43ig5b=@Nj@b?C~8HEBF(wjirmJ1>1PHsJN~bgn+Ol z-*Ry0m2ToYT4|7RL{3Vi|HfXbS^pzDXL=*3tUWG+jp)2>W9J=-?xce4eR{3d9sD{< zK>PC@_f0YGZ#~+qsdtLaBxcUFp;xd;0i)@diKsUbW@1{4T^~g3`(DR21H`Lfh*XTi zwKrqa=Pd@q%FR-BV~lpox6rEKl8jfiM!K}d2Ahn>x0*Ck=gxNuWn9qM5rrt9vx5&b zJRU7*Q5T%ojZKlnq3Ai%LdG5WgW*XlpYWEok*=r}(Dv&Qc^D4v)DsO`Y<7OHV&u(h zOsUE9Gntnj_4Df*isA<8ffZ@Xk_!>kn4GDsksGKl9}geyevza*q?1HIc{j1Z_mRbQ zcRsMkZgLZ>vmPdA7lESzeGCaEr#p!1P9;wUS4+`_SqwI}ecgXt!L-Z~i}C}}Aga?` z#BX(FH1}t={ht9rK1vg7M`+j|ZBuiHEsJ)GO|da7BuQN#=j@H}t{At&O)+o_vs#rs zRaAMUr71l(Hn;1+Gf!u`Vc2u1m)m`O+)G~rHajCAr}>(h%1rHzDahBaDBMO-RCwI6 z490(Ti>j*Azblf(x@GOWt7%9s)fw>KUw?0*ot6~ULOk8L)jMg*^Kt|iXMLqo6EaKy z1QzirDYhH$*>8KIK>T24=f{}NM{;AM^Fq~T$od7F6){lPMd48KBY**|O~A(@Coc~v zQDVdoyh%<@PRz>-bhb!Z7_djBG1rnBd%-|{FMP{wjY*h;h;2JtqyLN!cphEesE-D> zAC4`DVwT-qJvcrq^~?%6JH6U7_VM-Q>efJ zpK4)yK5(;~29yoVlu6Rkg2mOy*r-7}hSY_VzBlzcf3OXzlQvdZ^6beFjAnAKT;)&v zjxMZwRc6F9PdDO4Ah$~N-h(~tq-2Kn&JAt!Sn7%o^()K@`;^1=tikR*s7y!B1&I=K zrzv8B)Q>J`bgOFv)3kYS9ZVE_s=sQeAO%MU*j|FNqyI7X#KcifUqEIuA>sew5ln^s zshq{byCL5n8kIZoO`H8r$0SgkMB!nUGckY9UT*B=bs>+ifq zVGEP7@Hzzrh5k>vP|%7nw-=q+n)hnn@8<#mqNcS3n!Q-@r7OHOUwYWU_;%@}^a<;Bsn!C^s^j~z_wPedtkYw8>8)5Y zRAj^TJ{w*hk4z;UMS4zWY>_yO@P%n1>FM(|j?c9RmuC^bp_R!T;v{AGKg2(!#1FkB zF z;d;iX=QB|T&uZw50<4mYr%7leM~17r+I$N&uThA}&dx~)%gFg3d*j)`BMT!NS|odw zO!$lnhQmeIpWg2rKRUYfXG|*TWAYX-AfaFdC1@-2$&RAZ-7zd=I`>SG)Zf=_ zh?Y*DuUt+`EKf`#)Wa)ULf~((hBGr& zalrG8O0g*8LnufoPgmtZ_GzNot-bxgdtn8n?MJyCYUocz*Szk2e?>H;zu-$-2(D~W zTNPfs`|~dtYwYsfWSWW{S{MxABH(i_F1uF+2uWPOOAR<=WV6j-YNT3>2$o-_CJ%}T z@VD&6{+30Oqr~NUtgxy0?8~=GEMD)yZ*KKme!iWZ;=RS0=`C|Ltod}FMo*1yTLmr^ zs(bERFb=;SKE!`;~-9mSr)TM&WN2d(q_vm@4E4gMoZqjfxiSmZb~ zJih@Xfr~M5@!&F2D9ZjZT1e>itql<;RkYL4=@EK}uu5C@4iUj0_p{9s$Up3yK2=!C zOO{Ez-dNKLP)7oEOJyN6?AtF1MAyIuo&oC!f(&nG8djNi0+G4L%Bh= zl9ckfZYY-Jg-q^9xBdxFPN*?=)f80`kyziC+1Pv)Lqub5(e=#W&a8RA*w9L+V3P(%M->_~Jp?S7BvSe2d52 zE7JuaSjeYqXo&<&9D59~dJ#Vu8L+9q>`QoGE^_QlB$S@*fNTw(s)!@NIl40`s2)zu5hGc(Xo?Sq_ttS+=cNKoxa=AL>?Glre{0`!Wzjf{U zbimp*+3x1R%Xm%{?HSm^kJaXKY4rI43YcMKgdz_i;{k;?o~Mw40wD#(Nb#+21d;hZ-K}anWM3l&>Tu_58&r=hl(O55&+NVc3D>~;l^O_+Dn6o#6G>n4>Nd+u@eQrPQ>Lp|Iy z1p7H)fxyecsCpV5Vddj^bC2zLy-GpXBG_ z*o#tUAdNM{uynR&*K632^_LX1*%SGA=ph#;jbjq(w>+lH##`{X2@(>K|4ffq$E()U z0lyWjs5jeN;z%zwb9Pv0BPGvDxOya~9@hJn_f4Qdt(WkxoB0S)7Q3Ffp<(BgM}sly zrWwCkRUysXSi+>riADuFcMd}!A|mDCthBO}n;{E>D!j#_%^)s!ut)YMqsvyo*=}UQ zQoedhs<^?c^+M%pP~VcwVdk#*cQy8_4dY)<>T4`j(Y&5jk)k?jy`7&cg8dzmDam>V z4*Yl{{deZh6;%3v*F?qT36~#5d~XH9pEl&kCAE$#UzTJS{*cWLS+v**u;40#ooo%ieg z(yQqsW(pvNBl1H_)eN0l6*80lo?9Aw*dRJYLBx*+d~X)#%gUjc!eD%gTuQW`DtI*` z@zqfFp1t7IdZ}qgYCk+&l&9rvi2Dmj5YS|P#f8I%nTx?e`?5erY6tVoi94xd)EFp) zz;|+d(Dp!4q}-h@HHeGqH^((cP!lic-J3P&E5cOiaN53m^UJ4=4zJ1!Wp_sHF1V_Y z4a8?xCugoY&kB;fzQ?B`9p&t(QA9rAo{1d=x@KP7p5M2h z^~lsFtsslCZO;uQP4*}HVK_W`uV}IWy+M5 znFGx#pmlZjP=|(EszeD`wK1f^08zn89CzDd2w1B@D1;Kol}53s<`Cf4KmD30p$FZz`DXO*dvXtOk9D@s)Rq?srw~^NPa3 zRVn7#fos)v1#Z!|qgk5{z*HK&&n1(6v1SnX7d7|j)`T0P4KnI^p3H_`a-DD|Vu1x)`ys&>#+FjAj@~sypAz^+<<2pJ6+#8`-@WJtVMW@eZ=#B;fPL!KoJo|gr~TrPbw z6AD}{ttu@2+ATch+L<|&qO#3=)!cF zb4dNBN|G~IPJbRjmc=-{^evQ_F{-F#`-Jg6Gvx( z33aEqc0@w-GvW6GDdtL=*#(pBq00rqF^4v&1Bok_cEV_^tmv|(GmQIsA(6HqhP(SK z6wX1BeIeM_FJL{>ctlhv6Tu*|t3Cm@uKN0VAaOWGjJSVjsK(<_o05tO@PHrBA&@r`!U3(r@m-6}-LgaJ@1=--44( zKJJtWMN&jxvvh99DrTSVt59!*tu%brn?w;2V^5LNC!)Mj{NEZMWbz27oSNS}jY|Gh z68$8(obNVXTeDwvL8KpWvYs9N4`EiC9uQIX(wPPlS;v{$8kHmz6v7?nT7VS@S2V?5e5=Oa{Yk#rBMYK)TD9}G^CNxibS$b_tcv9aB}d|sKC z2KsoMPz7ivgVyr%d`!r)>1+1(MMvVBhzMbM- ziNii+pkBOe=Lcx%!8JZ8Y&>rsE^k6>Yd~+lu8vPze`6M~4tO9Z{I}Vk$Nb}a!2;$KX z(IZ6wftu{*(?bB+Gw|r*i~Zks{m*l!SOE|_Q>X6u9SJz^|7;in8yg#wD-9OnvS?cV z|9!Ve`1*E${GW42Hk>8PMp~tB$z#lv7(htJ={*5tr=(~n2yQf5aNlo<(q_lK@cnHZ z4n;Ayne!tFJGLMeNQ_7rnq;hKQSYp_*u_&ECmbD%J$ZR~YK_LTy(F>!IoE$r`2+}% z5$2lAD|d_jFvT;`OwrRtyop0spV=-iie|JbQGiJ5XT!235GCU?Z6EUgZJK0aS~O@t z$V_q+#s3|R@TQOt7XxDi$Y}gx`-1(pvo*!G0y;0Yw`(*7pZ}Z-H?v^1E=Rh^G@2 z*jK=1StdZ%l0kJ zRN%E^{NptoWhN#PGBTYc*q@F}*P~*?U8p^IC}L7l1Y4>_5DC-_RPfMo3 zx(zNu-Tb#%92BKTP2)Qr_gEy&Ac$_m#$+q=2ygLu{Dr)i!t4N>Q?r+YCQ z8(JeHBS14Z7=a;ZX-QrCg89eig&QdmbH2Y+%uv+_#ng&@aw8gU+H0U~qZa42(gOvPNKa1$g>?jMcJlF~Km8P5L2Jct)yFwyos5j0{j%fdM6)sVzs9;9FUVk@cE z)}`6{fL-=C+g%R~p7;8q8y0&Wy*22XXUoZ=R#pBz3z#wqZ1iNLG3Z7+mL>-BUBI#w z^XB2fnZ;%U;Nwk>X!MLnH2aTuwLM+#WWL>9oUgH}4Ni^bxtT3Q%P~!CmnH?i4c+Nn zuP#(wV|n*fF=VvcUx`1Ye1yvycSvnQL|=b6>Ej_>&3K;S!zys!Ca($9^9OC73WMy` z1Ei%mZYK@{nM_Swp3JK`@3-flchMMAbD>GJ3NIFdeBJxDTaK@CrcA?cIdha(aIG;X zPUMB&T+Xat3Xu{u{T9#rFRr#A>P3+4dql>rDcxaX3x-nY%Gffql`8pdln`y%t2w#B z3K;5tF&dDi#ct>4bj4T)!$+@+^U?HJx$LL{z1s$IL!`9_o-=E{+`$q>qGT``UY#=e z{3|dy^!nlJZS?wGq_5=4zv?+i&=~gTiquePG(tPgI4`EuCl=$9l28Gf7~tw7QZ>EB z^Q(U=e2NGbuIFh^Hg<>%KQ^lhvFEg(<>+f7QXu@^x1S1CSUP+`3WXp&{;TJrrOHI# zU8^5fufISypk|6p@@Wt$7j^`UkMyAAi4Znd!?q&C$47IHmY|(?i7@GZinnFA|=Dvf|qs zs7Vvrabl#m77JC$eF+KApGpvK^cSzCZ^sT#a>xFEDSHfDcKd==<#JfdTI5xKmBQ!= zQM?Ow*#*OFPI`1# zvzwBcBe2~h*YvFT%Mu|M;;7)NLl^bknJ2YY`zvi_VNp%=^)Nk{hS&tUWC8zBpi2_k zJryB?0-$hPZMxL&3yK6nt&(WF;sJ;;CfOyVX~R9axmoR^&E|)lot^5y`qLddl*y5S z&;6&~=SaCRo~g2gz|bC@u7^>Kr0wlA3-Dg&+akx;YKDOtr1|mY;7Im739j*gwcP_bW;9t^;N)aB8HF8Zo{Jv(ja@CW@J*T|oaST!| zfPH2F4}aB_E1fFv^|TS??9o&-)t-ImG(78L!k^iKJ$Dq(>icgwqq254=-6#8JAI(x z-7cW3aeX|vjn0XR_hkznfn0FJ*@vMvna17VkAHJ#?B#Wju;e6*39Ij*3SAJ*$2%{6 z!V3yU(#u$D39931RjsV2;^H+YWzzFiFW1}wCBTW)ZL=Ll{I&hj2QUBg)%zjs^MU-V zjuT&!bzhRN8Q_o(KZ+eSg%c=D5eAO~t6VPRq)FnEt-DYZ5>H^8*I6AMdxbJb-vUno z`oZ;Xwc2e94jp86U#$abJgN2WQRGc$u$n3p}L-}|92sVU>}?Jx8xmLajv$na>W)^$-+Q=UfC$vhy{ z16fcf=pWi-X>QGN2zHLy@zmzQAtta?2>_-gpg^px^8z;kK%)T)F0M4t&k+=)MyZ%1 zrmg*-33=c&tdY^wH#MUh7e27WU^_^jwc3t`YsHAnE zErgMeZGVk{=#Bc*pmk6V|1z6|Zn1q&o7?K@Po(L{UYNUozCJY8@QhEx7`z8 zlRE#itk$V;xZg%i$F#c#O-puhs}6ojBRt5A=sa;%mJa7kltucbLfsO@6&IEe+0XAs z9F*&SMIUvab8TqEGWamww#!Pn;g_=1ru>-_E+9y0@>G63Vr@m==uX+TEr-%4f;@ko zFMy%s|D_OBnObp#)8x{u`jC5Tb{k5WWWA)@X28juot+Krvvo`)tz2BYz4Sk5hW-J zjI!xcW#;kQDd4&41jKM#qj;La#kK6~}Rh%2K7nf}G7x6qaW+W`lgAG2@=|dFrcm z-9=zuDA%1boVH0$HP5?7eTuj@W#=11!$NzraJ%O5xH-ij42I+a6Qw3NLH?-Z`#S&G zw3Q(>hd3(_=gg_9Y&LHYAG&k5YQgP^L1G3f{43@`pEzWAX?)kGGaRosymZKHuWCJF zU`<;07X`lnX8D{%2Op|;Mrs*%XXT1>Ui5+m9rt`DRDw%pvbvsUP=zXDc8u?Yh2vl4 zEU_ct^Kag{liNc&p%Os0sL4vz5=CSpm)o(@%4efSu5lpph+yRy*=dNfen~Z zkj~=*00Hhx9v&Zo_=BI7i*1{P0-r!wwy*>b4-c@I9vm8~Gn*Em$AS+D3A3=Zl_*x$ z-;gr5wH06@<~_9fa9vYht>4eAItL>qK(M4_R(uB&+*}{;a+n76L%j3dU^Zra0%sW& zf0J~jIoc!xV^$p50+A)JK?p(V+u=2r(DFVGG?a<5rn=sVc=m0b8~3s8>4IQjV}%O+ zcUDyCLpHHv7~4Lj*k@1JL0HJCT=((6q_>!Mo@im}$&>z=3YX)^ih1+>hzebL^Z9tr zg=+uIAI2?|-^nSSGt?)RkH$+2*D4V*OD1dPMX|=T_HN9X5w|Go&@D?QexaZyOM+eU z=ix{(sSv)|L5UtLCQn4OAs5G%OQZ-5j}|M@WBsQq=*~yTM~IC2>yZ5LQX(l`?me(r z)yv!6M*v6&4MFd?bT4oW2l&sSh*rGqAz{1gYJA>`L4iY{;$i1uU9UQ9xOV*!a|9&8 zQf6Q{;U^fPZd}nD?w?Pi1NME@qT>Vv?Q)^RP(ryizya*}2|Ug=NWnW6-Y!#0GMpbN zfn~V#MAwCW%pof`yJS!lP`cC-+4vWj)?B6CIy4=52ILi{f5OSQq@dL(5)$eM#i2-4 zBolX-h7KW_Q3+~#Rb4rehS&%-DU07`D!b~a{&Zp$=@j*(h(SrB;Z_CgzO|7+IaoTT z@r%`z2Dwbz9f(B+9>uvj71aH(Glt&M!lTL9rZm>rzOGkSTEY?DgMjCOsK2adckd@zbW9hsSy76Uoo7E0>dVa$@)D?+hR>*O||XbaZsoYe6f~rU2i^EH;0h11%1L&+>-ndG&zp zC=U8J$~WshFB{PZcP+9r*{>fsTzY>K>TNM}e?Nq5$*2VfGI+h0Clcg&wxon^aXcjX zim6W9KB-1)sjDNGhZfypJ@xvg`xH}_43^n5l0L^PY9@N~sF0@6bWW!2~wANh+y*_0NR#i{c+EE3k@l(f@?10Nv5#<3lb zX<-rJ1QoOeKeveGC~0!A1b1g6g7K0?d-g2h9eBXey_SA%@0&z-g#A5^Gkm-D3J1^2 zp4*p~i^D9dc*utx6+1WIspAN~rusv(0Ue0Epv7H--bq~zPgUVW<Td2x9@{SfrK@1QlYhOr=>MfQ+M3H|( zeAfo?18ZA2NFAsn8H=f`--z!54K0QTL%EAmB$>A}qycZCvOl zhf4eDT?K8d!5J1w8ln3mI@tEZm)9oA>1{9D}lGQF%z@m?sK=6BBN)=+5D!%;AkXKJKEw9(W2mR!ufZ|gQAG{l3ETh~By`RdzIcBI1rQ}ai`uoSVgE7VTbtY~)Ah*crM=*yq7~K)&oP#$s z<{Ko${_uIr{PKLzkh)|VT?DK?sayKkhYS9N++Ki#=|+pM34nm1^Xca)%e-{nxlCi%q4nfet02ECe?$9Wy zACK1NPm@J}aUtg~Iy*|aawum~R#TRAX9oI4NzN(%)oiF;zHj18}jsx~m=YmiW z9n+8YhO4h_Cr0ilLKbLI#XV6n$qP$ku3JmWa#@O7mhRRF`32D{=3r&&O z@Xq5><5SY4bMP>_%U{X6hSTWrnEdYgnYh1zLKEJK!FWsgL)l< z{_eqGZln0PEp6GLIWgyypc`#7R!SYo%UeFi;v-1TKCfDY}}hI%a#2}i6|Zd#yz1XH4mr1sE6 zxSp1Ht@)DU`00Zd3}9Wu0?VRtn);7h)0oTAD@7%BDq?H2s2q}vZ=VrRWQ8e+CqmhJ?y6!Djdhc%S>Y5?J<` zWQ<0S(wN8#_V!Q^uH*XTE=ke~FOMRdUt9T^Sji)If5u?6Qm6BkG!8knct3jD>_C&a~Ugryk8;shF zOCj9WKQ|w{yIBcPw2=3_jVK#A52P{rwLhFczWt~em0tmDN$aVb2vlb{pT?@cjK znE{F#!9v9T+t-qeM4DkFFup*D(2vIvbz#`Ae9&?^j)qH0of?wmh}n^yd)N^Z<)Y&% z%m?cWv39nPhh`uWrCKfbqO=JD64J0-FzR&4!<2{+aHew{0cr z`|9(?%-@rBD&4OuB_GMJDZg6jy|8b$e#B0D=O!*UJF^?6>0MccVv8YfOK2Oj5Fk+l z*otTBqTHoldwp3i#j4Qf^$@RN`Vj@Zc5{eI!t9&ZoL{7*cN7Cx_675?-daV4m6)EYCBgWWos?JcUt!BgbWn6{y9}?p-Scu}ZkKgxB$SlTtCoJzx3vJrZO=#pbnSTJRqz)Qijyoui`1sCQTfQ;= z+t$VvxdYBWL+|bHzwRRYh0h-!tbXV>6k6~G>79oIl4TDLf)DR59TWj9-e`c+n2y4i zH1J-#m7bT}bx_%XDZ`)MuJ1SSqVES@A0RRa{T+BqzfLSsZr=NkAwwLKy@=u4s!FH+ zwOQ^(=o}+x5S%<{9T;Uth3lvcWV{T6i`DVv; zAO6o|&7ijG|Hu#F;aP@F#ub$;lz29T5(DxLM3>ZUsm8`03ODTI@0KL8ii&>!0pbHw zJ|m-|fTSLLLEo&F77pOR4m2_$L|$*NJ_7c@KYJFP|8uy>7wV|Et81lI|t&{GU@Zk{__kFhEN#F^ql_4k;%n&;}bfd@mm5 zpaS$__L`%L+1d|o7lwzAq=nFJt(P}8CK2$u{?yZp9oQviVMzdfrV}$WqZWCbye=9(N@INZ23>-zWuo z`>hI#okr@EPvs%-oMEN}I^GCC%;il3T^#wIgx}t6+?CMKEF@Sbu28%gtN8i3ylLf* z3>MlH&+K3_bR_FmgU=4K5_TN+>R`*{Ju#F)zUw}OY1Iwj5G9>Rn!RkCJAMV9W0u-^ zTu>dRSivH)7o?60-OMc8NtUo?srZ2{)wn|On~g<%Tfe%D?MkFzTZIuUYTofq(dz$f z3o+c&!Gl2d`TXSiuq2(7sL5G%^L>?wMb}cwyIaPJ8*xuJKc7RzKw41xIN6ovzT*I{Ve*!-&TLeWK91T`3RJIF#>QjY{tM@v3vJ`IC)1C3MnemhbWY)L*s4%&4E zqWc6g1wx&IR)9EHxl+Pd8;z7nBuw%zFHwi*&1~oXwVqeFGD)vMAoPK3d3+{ETZ2R> zZ8L5W-Vrzb5zusZKX6D((@#5U@!GlJFDDYL6!|8z7AWYyctw?$jK(&0r;kr0(gsjYZr(Y(de0y_&keNomZ&2TxAR=bp5BCR8ewpILn;RHlN+{3_ zPOsmSm7Khw1go0)Ok3OAmGtq~DE9SldKTd=cdt;GapBWX{<-j<3+mLt>z5MwKfRCS z0!y3ykM+O83%WM(%j{7yQ==OlS!=sGQwlz{SC-9ZWYg4Ex~^;9DMrHMK21PvdES99 zM3tb`@KJm-em(7DuWtSnmHtxG+FUIv#pQmCb=K@OFTWB+wG{LC1(hHGRe^MOW_4ZP zs5BWJ0jtw}9d?{{Edm@}0>Ox!ZU|k|eDB`A^Yoygip7}USxWe^YkC~^JKa%PVd;^Z z%S!WGVHEN&G}p`Du;fazQ5-!#?7Y*Fx5l;e1!)JW$Nj>}J?2bTTZ8>n1!drKtI~T7 z1k^jqF|$&T-9`Ft`#g%{^DT&L$V|KjO^)~N+5<}?L9)=?22#A6F#_fx1uxl3=%^i=_w~j> z*#jfE*a96y(q@@4&#iF5#k8=m6M#04XCkDxxdrlEr@k_~nQ?7M1w97L6}`Ozbs8#O zkw|>|PH%kft35Y7f~Ip!Iry}_9}hRT3eexwkM)IfG1I^f9C^3*wk8mmX2i(=z?BwO z*ZUfO=XafLZG)MPBxlQA+>nUh+TIS@U!=vibK70}8MclQ3HTL)akFmShO}hxDUItY z%k^f$E~3u~?ngiHYRmb=fCyte@H3RI{i0+ackosXoUt3#Ww7~+lngkW!0(`$d&|Si z3ycLZJ{yzNZ5N!GMj3|D3e9XZ(vwV>X>wQHtCzx{@Me&H3VXS{!~^75rnr zk>G;~{aamzs#AY$;^14J?BvHojPE^pj)y5FSEk1^vX{j{7Uv0x-+4vZ|x)vdbIuTm4|!}a*QDSUcSRb zJf-MPqbiFr2aFxL|0+xNsJF+C9zz}!v}6`;0G;bAdcO}6pr_jsAMt%MNxz+a-=Z?o z6@Miz%h*VwgO&!~XYq6aO!?lN)FA%=|5Rp~wXJB9U&{wSYZC0IoaQ3=0lneVJY9HU zu%Mq69YYi$#o+;zZjcStN^42fs^V~7C+$*kT56=J2)&cX%UAnx+Gz)A=)2zwndzUY z4b4}vqko0xa~iK=Yh^h&9tvKUFGuvDa#Dgh{d#<%)XECOSNYaaDItQNibD?3$G%?S zYya9!fGMF5`84xw3_ymqL|E|aD(zKHwa9O`yX6>L*CpwQ$%F*Akr;r)cqMu=1fBQA zhPCk}Dz1Jee=_6xsj+@(Uv_H#(DK~w-VN~tRG;LRFSd*DkDMND>9!wP9LV76P0lnz z#xY4EnUS4l<243QW8+&4j0J>0(LAy}Mj@IUdwA5RIWd$g#-ZG+$AZ|kV}IPhgL;C; z=O;2hy!I)YB_US|_uFAvoYdCcnKq|&q8+3OGSnnHjZfgYhprijJa_9Lz(M9rzCQTO zh1#9YAei+X|GS1}i%gR6$DaPO4IYG%6OT(s0&jFspaZioy1Kf_oHeG+30unvNd!^k)5jImb%pi&*#(@IX{0 z;@9NwsZO1A^n1^WTVnUhw*hE02`-I3o{}LNi#`5A&O9eE?)Rpr2(h;0p9&B>cR%Ri zg`xexX1cZ-NaT+Jrs*c!tP*K5MQHJ}H9Tr~oklM)FXajNOs_6VymP@z^N__T;Ql@w zHs9@BY9Z@4sa`5v7-nf{j@QBqJ_GHb(YZE#a@%JzGI41w|AD1n9bnDG!o^neEFNj39(K*fbubKKrL9eMY}HNQ2k56<3h z3uMp$;Zt6iH6=wSz1-VHgshP1#yIJ8vn>}v-3;zDoa>)y`T zGf)ltZ(7>z^+7{naWi)eMW%>+?`5qmUF{K&q?cVMx|=JeyrQD44=h$>Z=jpa?Vc+j zg8P|~a@056@52TDvdHNhS&em(R4`k>ebl<Fggz{H+> zh9T<;s3T=gSF{MK+MIg72)LUhaW@zNN+Y;j3)Y!k5{l8U_0V=AhcesBZpnjX>yFGKfeHk&R2}UqjwalJp zI^pWP3-rF01vVLSI$lriD_K5sJa6~le=J&pg>Vr0G+k!#{d4kICfN6;Rk=gtP+MhF zwX)_R12|R41cDK(tf(JpL`XMdRdd#sN^NezxMVP*h5zh;9(qcVg1)h;szgcmh?B9r z&s#cFDr`Zp?0Pecp?LfmpU>=86_vBaeBD#1k^ojoYaRNa^eWubpP8Fk1rY7RYg+Izp+-s@cvC%-!mL$bkdoHASfKtQ(+9W0~U~ckE$MB13d;?9hXN(ZG^ob~?MV zg_OO4g+?n`@DHvZ+| z6v1RBI^B))MZzu5HtnG7vKx-v?f6=vIGmz^emcdGSyx5nBx@?XsVAk{_DL6gx# zvxYO`5O(IE6mdU96)9z(m=0Rua^Oo3XnsKPj0pLN`XWb2f$6eO9$C@p>S1N;(TWO* z&@JE`HwF*KVfoT)FY+k+^XG?m4^VYMW@l#yM#ZqYoW(@rb0=!sWh?ypwfGx{7zCyW z>%b_5elI1(g}m6KetmQxIqj0vWkR(C0sUp?Rxt4q`I*H`&5ssO!M?>+aQypRWoYUX zw-MSyv)4}$F;u*+)yW*=ApaiVM@c06WQfx@79D*pQDSTafU7+0@uv^~Lk zTCezFxVF2!s(M~*!EYuOyD5C%PPrXmI-Qiwa>1l!OI6FseDmc1abWLKNCt0cDPb?( zeUvT+L3z=h0jigZXOcs}3;j5ZL!LW9KL&C;ZVvpJUyUW1*y%AHwTqe4w|r1UO%DAb z74oFv6FPMJerOj*{fvd2=T4>^p*#4!sXXPpSUxft~rN#g3W0JM#J47#h6CiO9*6yv}smq5{dz zXubr>*6+j2o31^zpmv8re+pI4(*y}PV`SLhiB}FE#_h914mzK=`{Ig~;^A?oe+&>q zY;1o)5$@E~)1De*wXNxSSMp`ELkfuiDSa{U0pGUF+=5X$jx%U3uP^M?`@rPKRjfs< zdTol{e{ZSZGiNYe{|nP-HeZU51`V*gVr%()+FnWLiYg`iMyd%4TlFm{Y5vwitw8b} zU>Hlq)$Y?a^8CiD$u=wh z`*%Ehr;VwcGiz_|Bk|~|H7GTXlWP!g-E?VxHxbL2Df;rQRqo;M)9IMDOE&YRg2SeM!Rp5 zx1wsPqdkXZuoCh4p!G&Sw|IMYJh*qJ8mmGun17&If?wK8N`CMoNyPK{#BjJmC ziAg3AO&2NqgM6eR`gaC!#y0;P)q1_8r3FiO8ED8Hx^z|1){X*-kU;XPUjPp~2E*21 zFj4Q{0&Kg(o4H@fu3Q#V`?PuLw%7|&=v0A=*01)svT8QF&e@>{g0jZ776=m&WaZZe zlU!kZJP~iSd&doq;Yuu3%Thhl*}&PVk3%Bvn$`pL z1b#CM1W?al|HHv@5HYSUAENGem(p>MfKn(kq^jfnKL`>SPkqJVLSThvd?WYp*Z{lK zr0FeExs{{;ZtY%;mbz|XAwq^1>1E1~$UDFl{M+i_P`S{>&3ud-uc@K@dWFdBr-g7a> zCf~ds87YLBsD?#W@HkYUX&Fz2bUTAd$ysZpIXeags82P(G5=4MgFW zl{8fZA`;OXkFF^r=!(uWWOWZGq!CJv)kWxdbIzz2Ln8~c#;57)jRn&NJ- zHE=paj2;wI>>{$|xAJXlN+YOz;BR#%*-&(rBXXAJazrM#oU&5gKWqlARbRP0EXRH} z^9u|qiz!)n(WOr;vS&S{NTmf3d)px zZVUea7_36D!N;lhoub}A*eF0~|4XO~DfX}Bw&k0mnq%K8_>c<6%w!mfc0u2sgHE}> z2hUs0LJ*q92hEi%(N7H$o24`oG0hr`_NMBgkI;n6vO%+_24mFhtAq*0qmQhV_w8kv z2rMK;5(j~>z1k%f6sDw7H*$yhTjDQ@VP6TxSe|75;4iPKVPK#Ls^2x%b_GaahOlHy zwS~EvFr&m9;3xrY?htymj;L}E_$!hT0V15)P&XN=YlwBxKpi6$E|0Yci$>M}(Qza_ z5sBVM<21}g^!%V^ad`&tk#u*z_PYI`Up){MgmoC=(*Jh^wL9V)11EkM<9G2Fe!$?g zBlZ;1_UhQgAW}{tb|TvWjawS7CkJhwjKd}1$s(+THhQuv{)1_-sqS2U(3Y{N22S~# zFS(z`6$rn)B|O7QM&H8t>k9Vhdr=?395Py5V zRs?e4fz7jO?N)$6pZ|9;kM9pfo_O&-T*Vrd=pl92(6fRl9Lcy&LPITEryEgs&a+#W zEOQ)j+qyuXU1E#yGD2{&i(zIG4W~$B>g;1^Q~r&Z?%bY%GXlf8InSo0xG%uS39UA(N41|8(p2!H(SQ6bn}}EG_j5iOeE0lte4!K zi-Hh1A@JBN5dz$Crxrjguu@#0Ldo~rf;$^80()rCT6M3DguEZfGqbfB4>qv-K}QQF zBYw1vnf+9=LE8qWjSb;0sQ7sDO!!xuNgSx%#I0kDI+K4FJW41=Ss70cw;`}Ls=GQF z83x{lCr|r>SkS0msj*#@bJd$T@JIzGlO(tGj=W+=1rkHsY~2iMtDg~$$9iMY8GZmv z7$i=e7+LFDUHwM^2oh-JP~TEt4PTUI4Y7F&o43iHCLhk$7D2YNBd>uXq9Dm;Gjy4sEuDlZ*q8?a z7jVI_OdTge!FqUjRO$ET*mDIFhXWm0thO5()_A0-*-~Ny=Z_6W|KP6L_G5d6vYkVL z+M9OV^gbqy(9zB6hGV{bt0koh5zffG;umYB58$Cabhw8*cK6dV-Vm>-j+LDG$V!%YYI&}Es+3c+^<6+@|b+LNduvpKtW@8CIOB( zm>7i4PuvF8n7vD{vZ!&Bm+;D!rjublQfy|6BOB-byt@+t%e|~<#Z&9~;6To1|J~)s zLg&|O_YR{UN%)T#-wu%Hkyz;0my?LwJt^JgkKkNb7Q17w$C2TX0-&2D3$C!)BK1zdV1}{~TZBKlk z>-*v3hC9Kx5S;W!8R}q)FH9aXmJG_AXe6&)&mwbXZU%Hw-M2Gh~AAew^Ze( zcZ{62-Dt(M@-uqf0TO|=grS&YC|ID>e?@iq!9Hy9b8T*`Sk}Yo(f6#vnS&3{^vY*o zzV5yO5e(3*9G z_6W3R#h9((eFB)^FL{3kxM^^#kc1oNQWVpZYkPOC1`0g3%Pq+2G@@-5oT;*USV77izHD@c4jh+I5 zjY8fDzPHxO_Z>86`c7gWey3AwDK3JYA}WIpvX)0$1nQGQ?!t>H@vx z0hI+uAn=){A#Eqjde!R_T31q-zb~z$4G@E_f!zF=BJpvJKhQBGZ*EAFkOEq@N3fu3 zl2?Lj&a_})Xt2E=G>E6t`v#q8G#Gx{ES4_l*Yi0-7N^ISV6zTnkNb;9`3^svw&qpP zHaEMRbSQ|3fH9^;6%>#H0CO1}m!=TAetu6)zt6--p*Vq`Y)PfeUUvfinee@H8SgQWcEnm}!6Zfp`xlLbJC88Io1+q-Bw#{n zVzH2sIBSa}h|6vEH3%nSVEpPhL3Kn#L~PrsGw zFNv+mT=CztCyC-P9qdajI%z*ofT2XIs349o$e;{`Y@jD`HHTjZupr<`C)CBj@BktW z3{Rl2h+zPhQ9c13^Uy1;vL-i0dTmpk26(#g#u{LW1$k0vvu9e5Nfe;t%=IXJfSGlO zeA!5F@ryFQbs4QS-nAN36`@}c&hmCh<(Nv@v99k_*6V-6&2dJ!RAE)DZqu;5*24?J z$z`Pq{2df*4b7nf3k)(*od>|k@fjiiJf@Muf_Cb9g-3!zo@e2A{_6o0%xvW*VCX~! z8;Xr%|HhpyTFhkLSzsk=_tZL)4aLdrZ95`jr1x$>APOY>Pr!$uNVfVMSjpfoM3W49zu*ntAv`l2NU~NtK?oDld`T7>Lr59Iq~P&<*7yw-=$kUi z9gh=w(5Ax*o-kicL@YpPfouVZ#AputE%9hq->MRkT6|6{Wx%nl7}agUJW=)qmNy1g z9S7T)-My~oSIN&4k@5}#0%|BNCal`e(T^n@sBi$5EpF0mU;a6Zw7L_$#cDk{$Weez z+(r)3KNOq3!b2QD%*h*ts_P&W(UP{4TmTnI%CONq}S~TE-E4i008m7Gk|~uV9u_nf&!AdzRzRY2G@6L*52l-UEin{cRUK&(7%YsVH_4CZ`~J}%2;mhO<<-B5;PKi9&j`tII^Lvj3ye2 zKte}%4(CPkH2WQE75+YF2l3%z&yyX#r+=7K$I@e}qxpIE%?UIVy-GHySoQ>4;I`ZxVt+Whv4q+?(R;41b27c=mYYfb5GTMuim{= zwTqrLYo_g6(qA{B^0HzGuwP+8KtK>A#Dx_>K)_)@KtP?LA>SmYG*F-4{=nFXtJ{Ns zAojigKz~sqV!sKY93*5!pf;h=;jp-`A)cH*L?UVq0*(%*)^;Gp6=ULWB6x>4k+6}y zft{I+gPFAz2pS9_@tYj>T`pv8<7#JSV(I`=hBU_YCPn;^+8OD+s}eYvSsH<`-XXoU z2L$H*iITOkgR`EU5r~$}>$^JK$H&d=4J`EDRP8_-yXvCe#L(|zRXsZ^GbL~6m2Ev13!hd{8FYjh`OLaG1c^B#x*i% zd)vnaV#mAWVk|+fRFNhFEbX9)+TF{{A1>c2Qzx#DKtr$k=(0fDrnsQHxvx>GX3AIQ z2tB4&sv;kqxQm*Y)k!Op2!~@gGtNCDe;5cNI?jER+rsVlcEgH#O7SBj@Qh)8N2V)RBo=P6Z!GdnWM|iTds$#Iy+)fC!6_&hx*fv zuTg^yH@N2O!SZ<*!}&b5+0Wl3DonVg&b-64@5CpzX4`ftRrqSrbV{X!->z*Am%x|+ zjBPtUnV|mPdMJm3qTRHYajju(uYZ58{(y5~>*w_>TR5?!gWQt|sd2VUF&T`vRav}% zTL@KQu${9o=B5)m(-Po=gUeDhT|w+_Rb@;K zfprI52&>?i{?pjvSVdq=CNuB=7tW0DRcL#jr>kYxXDGxhO?74DauA1k zlAoM zXTP#)4Ly5xM6*ue2v5oJ4z~rojmyx{VyP^uzHP9xw(>x#My=mIx*u zXiv1CxqO0)p5oJjtpr;`zN>|*xJ_C?PoLC)-4>L!;jGN+3fH9l&{YuH|M;Q z%nZnNwY*qO{)m~n=0tFxxhGi|_%c7h4Q~9DJW0S*91OG~q*v7F>XhNo z5J&BZC)*d9RW`cq337m;k3;1sajIvKN~et^v%YP=OR6KaU72%~T!QYR?a#rCNq!G6 z_^2lj6!$xxQFiatt;fz+tk?6ogT5R}e9^Y`MaXL9HH*k9(XI#e*~I|hH#+&)*QQg*@L?ueMzzrH=U1+#UGG*k%m~|+BIw!)`fLkleD)`6DDTHduad&Trlyzm zVi8D31el5n=0%q@aC*+IuLJq`n@FXA@Jh7_{*|Z7CnHZdUp;1v8IBe_%e;-618I)Z z3|hEbh8@2P9G>YRT8j;StBk=m*Gwa&agnES?t(7qFKGpfDK#AfN4;0B@pjwy$U5%6Ckzv(+~1n;^_E9j z_1Ef-B`fV~dSbtxo$>eda%jM8+-Z$H_9eQw7|g@%EjLFad2G)vUla{D zDg1~ccps18J0@`WU3)doaA%ATv#-7zesoA)&?rOz>#X(7nD~ZB6M0ElD== zo!#H~5gdNMyzt3`Zi(F7>1@GOp8|G77!e^}1^IC@{{z)yqgaUdkcr|-JQF$k-ZCPl zkpGCVgeCmm{ay&XJ_eva4%h$VCQa~IaKps{mh3~h5`*5gRRI~|GvG8!)fZ0*9CD!{ z!cxOu`{tkpFJliskxhH7ffi?Q&YbVvHiRR4q;3)TP}t7|*7cX@Eoycj`TBF*HJ3Ql zsxfs_cC*+%w6_1a4)^RZ@VqRMrG?msz1KSwn%#&U`cWc~M1B91uwis!nFL2t>ybpq z>!4AWe8o$vd9qpthk)L7^?E~QZ425aLT&TG0Dyv1fcD?3qud^`Q?)Tm-kovIUbk^9 zdCoQYlL41oD<0n}91)z?fS3lA&k(OHV|^#9V;O7|#oViw|K2RMy(j5zs7{mpp#dBI zl`ja!DB5bJr{>q>%$ImZ$J@oeJ03ZPThsw%!Mn&m7Pi4mwS0rUNIVk!%41%mf#H+@ z{0_&?u7o`{v`iqnF*`>Rq}m@eEVyV(;|-zFeH+ii!touMI7u@PH(XCf|ABf9MzL|j ziF@d!6sGW0mgM;P;V?&cOm0V_Ka6`#&hZ}uf!)6`Pm(8Uw-RB$a9en0d&DcXKX%_I zS&HSp`#DJ1-NkDW9oo%Bf1D1?qk*sU^E_S6wlka!HW_0v+xSd9E&ZC?dfZL!@9)2u zOr`v*Qone21Xqj{Ycd#+L|mPj$f(BbI*z*9NKm60N94N$iP#)IS*$htH>|?*S}rZF zOh*{(*B5@kU~tn2osRoEeCn7x|1D!dOKFoikgcWBa=n^7;qvKW9vDKW_KmYv2}~fzonHl$caT|^GHhS7yi?bbLZc@KleT&Og32B2B&6dZh!?okn>AHH_T9s$Q8eC{R@4Di-1a6{aWG5Omt_j^9H`*?4 zmtcg!rPDlJfy1|Nf=gF;WOJYc^p}2Qez-5YJlRsQvjQVMyt=ODN=sg(=L^E5Tc_Rc zcrP6P+{uk>ZcG4iZp?>wiOIc+b?j)cMDW5rusz>Qb)02k97k*HTd0wizT(gc_=o0z zm|??S42&Q>j{9GEgMUN1eH8-G{tQsplYdNYWn}u~o~#r6$Bi8bMlnFmV?ooBL~_ow{-Al%!^SK2^hlO#{#>o$)ZQ zAFbh~gG=Qpp|TVul^JS%vQj|rWE+Xzzc{X7uaLp+9O>k_6H92IHLx93+SX@IR!Pyn zX~DmoPmj9vF10RX#2manBtok6h^^5x2nBMx>@WWLJ|ERH$}@h4?vasc*_0Se}uhyWH=}X&}(C6 z^BJM?jwOjau7a&C8eOC|_5rtQcr+eyTN-v;3v|yXYI7{qQun4;*x19v&e!cf;jMnp zVl}XRmIquOlB~3=w=r*NF6zdGok|J%MK@SwqI}imuo?&lLS)h*8fn>*SZA`NUW#LZ zj5fZ;o-*?9NMUGB&T72&Rg5`WEY~N))K+FO9qS9tvKf!@_(NqhCF;ekPD+1ubvT&h zy!so(EQziE-l??MNFs?UVDO^CPm4s9XOR}@ea-ZpIb&1t00$Fr59xu(2vl{g?e0Au zKgnRZ$q-cDZJF%hwaG9ARWqiq`}9G1+?pB7mNRK^jGJt6TV&I)fT%wh&A-M@> zwjJdJ#Bixs|VkIQ6(8cD=m)T8jRvWqT*4T5#9?KCt*r_zU28!IJ@vq}sk`shK_ z-9T&~3E{xe^!ixDygr&oswXDcYA;snWFIFMFwY7!sC6~qT3pHZ3PENx0wNtU#%?pK z((GNG+tL_ihZfN~dX^;_;Gx7|-jjhL$|GnDI_z-MDoSQZFFe*BvcPc>W}+t}34Wc` zbz5K)bUs9x2iJJ8*ZiPKrVXd{4?n_Nz&tU0QCVHF9KusY#Y6#{vE54@M4WUY(sM*WUPQIqI4;BkP0{`ZButM{pif<_g+w7**4I)luJLN%ws zKJdPo8`uiO-@08X2Z1l=i=x_FcSCD-i0$>JO&YWew0B*LM(H~q_SoLMvpJh_ZF_ui zeOj`o9!iThf2RmyRJT}{Y@2JDCoQ8e7H#F#SN1V=GP2DCKAcNo>^zNx&5BRMs?U>v zk?feEq2y!pM;eeOsa;DV@R5DUBDsZsq_#x<|4lpmxAXb03E`+G;X zjSXkDS+SB%&W-u*Ryci+iA(Wj7O|q)g>H?76e$6=Mr6S7$%Iu9{(wtVfI>dmJ!k|j zAMa{cjPec}!f^C$JLq_S^mn_g3`;QWZwAidTKOW&T&MFCgT6-%)pn0xf9~g2(}RCc zKaPHH=Qadq4@vjdHI(6FSbt+3lV4gg>0Mm zuVCEOh{vf5!6f&p=D-rQ*30uP%{^rOAF|FLtCN`sfV(zMff-QxpBL5CsBK5^Cg+qw z6fJ~*AJjjFW1BmSq$uY9YEoZFWvg(u6Z?fBm^@W{KYkdjnXUlZ)*wx==^ZvwU^TFCWu}+C(M`rd6 z#`NuBMCtdMBDsm-dx>3q&MvQ(n7(WMGOV#wRJBsv#r*t6rW_#8GKxdW<00Ii_P~d3 zD3IkzJ6Vm)@RIdokg3<*Kwi7~m`%xBp*F&wga3FCq#C8kaa)_Aq=X05-LcJJgCynY zJN_n>oZ_;72lIR;u{5ej0uA+yja7Ke+-SKrf{T@7q>lB?FxYqQr%91&-+*uLZ4#C> zdkc&I0_0IT+1$k*4S4>g9ch?Qw-OZS0G_KcW(Nlc_!8YEwm34G zoa}7SSr%wFIFHQ<91S6HVapC@CBO2h-6CH0?%%>)&q%w;9*(ioW{H(+nMM*^+i8?A ze9Yb3b7Bwo{V6_S8*Iftf_1c#v)I0#Pd&lEE`4S&vk3R@U=VfOzgQBheX?@?wi6BA zI2OTN(HG$0dqziN2Q(s;mPwC)L4EIq>k2nREOMxD5}zm6(R^iLe*PzrLE9u2Gt}N7 z1Z+$Z3UuHk)MeWl?oOJ{#$|8j&Jgm<+%DP;sNgNjW4ME-l_$$+{FS1Indwl}+36O2 zFV+1qw^WETK1(bz1Wz;1s6AtD=%%5I07&9ug{}1q%;nk3YAsPDb_yQc z^T?>g>8Z3mYH9$75m_7@S)%4IT^DCxdV1Qalwfulz*U7UVgrojQjNy}ClZzA%T*7i z^8Io>&E(SrvOZa68XEbc`Z|W3dmWgPs7SM!Tf80A5IXP%Ht}jghKbKspXpLWRM68pxu?OoP?jBQ+ZDKR@|6fKog> zndzkj(smP0(0tTQKC2+d9yYiooyHME0Bve!*4~76hq87?z`N3M`%iaJ{vwB zOecQ+J(u6c=J&4ue1aZ%9f{!kKyH7x|1AObjRxJ|a(h@fS0Q$II|~9dtKe^*GjH4D zC5fz5%54{c6}W)7eXEp(c)(>y!M_!Jnq;J#Oi#Dwe1A=u01VhS#xuLYCl(|pY++0P zyLQt4d}VZ%iJc|yrWdC&$FlMFEG}fo)MDHsJ(Lfhb2t(ew zpPo?PZ}AVZVyJk#kL*8N1AnJS^gf)`|Iv*7cknJWi$$(7iITFedOay8^z_6VZwKv` zi1EQAXqNe1nO7c$6m(9q8=U{Ri6;U^qai+(af$YjV#9a##%1fm)R08yU51x zU%>o}I|v7?QmB_eo>r$5B4}-wIqM}hlMGj~AjYaAyQ?{x*Oszx_I%r8jJ6$YzwKCw$~6y*)TmmL@a!}(pBt2yJMlCaBuc5+Nh*C$)s z1nv|tsuGL0WU$|5bgP;bE8!%l(4O#WLhjNR28D<-@oXhi@2`=OreeNU*1Ox;jyQU2 z)CA9!wDhdu-bDI+N0jAKI6i-dLawM@3V3bC;Sq70XN!Py=iu`jJ;}A!JqmGsg6*?^ zghb&i&1a`Y8I$o$3iq6RffJ#o!=LeXid9I=}W$D7zC`T9DrIFV)!%*byXVuxtTCUb6OFVdU1A5B#KMXwL?YPMgNVppn^hf5Kp}T5iH97lJ;Jg(5K^Q`Qq# z;_8BmRZyGVuQrv5;XgUwhw94WK{>y7OZ-BMXU5KB=-!J;=Fav-t*N|AC>4&w7VXG} zIh0AH!_6E6>6G`n`J*SR6lEG#%{vM6H3G9Nq5b+v1+KcZJsoTE87vD6r_%x5q{PJ1 zW;U1_RyFOt22v<=U$OKCY*NEQM1+2IZ=TSl))2;-TS{8`e;?5oFZKnEPh675LMNoS zr&&2s`hl^dHJd|1TV>(@Aeq|aXk$odWy(;qnoX9&`T_gP0iyuDBC)-H-lfZ|MeZMo zmqzjbmX!W~n}-e6XzOx2N$>0mIX7~QD&>;Y8WMYqIlr%ElZf4&RrcBN7>vaSh-4(d z=sG*7$0xR&_@X-bmM3RDyGzUPXXeuO>^l!PPdfJtZKsMoRIEm7w~S5$Yp@bq(Tz+; z@ICt~!}emHhwUAnUD>r7%d)J;yAdF_Pqn1Z4vFE2_APU-uDETG;;(0~VjK;lD5sFE z&9SwdPkJWt51G%0p3SJ+;aWHo5CjF+@?RUxhuF|zt~iTTq8wC3)&a)aA&-K5)p*u2OzfYfbR+5q;WjNB;+3 zN05h--a1Ag4lr1Q;ug=Wl@AEjFm3NmUU{Fm<=!OM7`vqCviNhg2BFT|h`RlCjoE?E zqYv~g^0Z|yl5_hQO#e|>kHH1SB0y(vHkA&j$r(47t|?SVEMliK&dQcgg_hh$Oml z1{!)`^7XylVlUmPH7pv~&l&WqL}5>WYGxKTwv@>P=Z(-*V4lbywz;sI__sV@u5}D= zyYr7KEvu%CyU;9sWinC;8x1<69&NE$JgDtO7}mFWNrJA|6Ia#w#u%X0dTD`3LuYCu z9sp|Y+#>uFXM%#4uUU8f3ru=+6|WQ~pk2PB@z+b|kN8)YtgSr6KIhn4F!F zn+SKCg(PIdH&~svzGyD*U{T0LySURPm?6920ZGe~9Eln9$v)d}qdTqFyEBx8Zv!cN zD!{Dlxu0oylX6ExGc|mTK2y+~WG$%mVX8|WKT-K0erWkx{XiMBt2E(W03 z)bH8Yvoh}ROdURw&>6N7$qyRcIhmL$sK<3e-qBV$@> zd#*os-Z{@)jp!uqO4F9HGlq#KcQ?WIBJ1KQjqHnu@KQq3?(Ww}A)}E(PSpr%Z3|tF zb!yWYxCqJy43JCP3yG(KUgrJ8#IS2%u za7KNKGG0hx5BWRbD{XdHK%=9h-c^g>R54kVN38o?!V`ffY?;{~WwZ!`Ss>8I76Qq4 z$CF}IdQ^J|q=`Dc4E+lk2?I$e6XA-twu4fQKvd8TnDtLIz+6}317Mh#t*Xu~sjlJ9P(h=`rD)4E5I*Lv%$DhV{?1Ep)+?t@&nhAyz6$> zB+<0kRMD+IoqZK+Poxc?qZ+M@!)pnWy$59H-W0xXV>QCo45RSU?DlQPvbca-7E-S3 zLuYfcMWxtj$I53rP|-T^-A-qDXu&pR8HeXJ*{32A+kUo(MiwfU?5V@C^6~o38{AS77#~s!@RTIX`^y?Qd z-S7QrEU}m7N8-y2zOl@5M|*^CL9nuU_Wm`Euw09f*jSAPf{a-a@n(gtQ;KWZ-3y*N z1bKO0YD=Z*Peb$G6DFbp5z=*)YifU#2Y_D(vzHN2$Cu7;GXY0l&}8cu%1sZydXlYbr@8N^8 z2RnV$`DjlDq0JTITT;ySZ(*_CY2s8fGN*@HP31650s$pRY`Ww!JsspjXF(i%6GmH< z?IrM5n3AT+-+UVDVr(wOH~*j`>;LJ8oXEf#bi4YArF3#--sIiqN>44|vFUTIB;R$6chZ?gWQAcCw#_ltYw zGAMK0!XL;ycZsz%kxM0|b9^1CwH}CH2B=B&4*&UxF&nRaTHQUn+ZyLmG3r` z*yf57Nh>MTUcA1Wx}Ce-cWL_mD#2q)p^(fqDNL2PmTYu1=sSzTS26dhiTxgE!7+O! z2q0k}9ez+NNpH6|ma*~|QhZ+jrWM&;%XF7aeA&7RKNs9@?-`k$q3l{rK$@>AdQD4p z*OqHAy=;^Z!bOJ99FhF_LPn4&yrL)Q_~S^!tq>2SktgAb@lXXWUFj7)k7d+fNV0Vf zvT5kI^fc2`1MBLw!%vJ~yW2G%G^*Ub`Lsvz<$8id^?Py>SsrZkZFcMIRqtA$g1V6DB3VijB@sk;nG|$dMFVK<7>1P@{OYp1Qlzb7| z2hB(}7?vY&11=PgQ=Ye=d9T;Ncbq4(FV~KcJi2?jNQeAdOVB3Q@Rh{X9AKbiXxX6( zV3Hw4ySn0bRyf1VvXc6jU}UFQh|Fc2Ru&jBZl3rXKNntSvYhP$h0&x6TlR9xeY z%D&oD>^vq?R2~34Obsmaf1z`LnW=nQ#J0sQWeQW)M+%3;X+aM%&{G3nQx{~AM2+`8 zA^N$SzyEl^^?-k}(umkgK6HrtN z^fY>N(5X=;ROgYIj?6|O63Lc`%M{5B``xVu%x4zClgn$PQ5N8}W4q2*is8k~z~fxc zWmPNrO}{VoO~wKJ{F7%}^S1u5y(EHNDXi)p&v{ZB$V?4{BU63Zb~Vuh12k)z(Iyh+ zLo1z!J8+3)$$IF(U2sl3bA&a(3E z`e$}{$}&^JG~Vz9Uth$;W&Fk|_ap5dkg?|8>JeO?B_Ge{5Cb1e6n<-pW^2o=n>l2A z{V(~M#XY7{TR<#HYI7TaOcE)IUpp48mAL*BoLu4a1f3gSYmk0z)YbGsypBrbfmrDG z@~o{x*>JnKKJtkkS>uC;bIF>~S%?^ug9RtZolHDQAQacW=d1pMKXmbuF)Hn+8oa$d7A(e-GkM(A z%v_iA&iYxz8}ovL_C{u-Jr)6l%_eRBc;cOLvrGO)Qg0mGNx4!=Q+7Q>jBKhH9Ere# z>cpL)scv$Lx1Z7HBIC7mc;(Ey_7_52w$Ee<#7XX~)63wFck#RqVBM-%67iA?+y3&a z@Pc8v6eDkT8{-<@YinkRDYlj+@QoDMzB#^@K|Oah6!E16)$LkPP@u!No3U2bQK{aq zS;tB&@E<9&OYO|W(0}uUGseZ*0zd zszHZ!Z};7_$XNE{-A)BOQnhUFp5?r%wo=xe(o;w4OtDtE&M6GXXiSY$uWUWGwo1HK z$GqsU&U9A!cdguQP9&t)X}0@WjK(w^DR{I1^yfG3*!T!mWacBHrf!Z$g|UKM%$J7C z&xl_Q;R5GWZMEOl>oIZPB>dXA1io#=SmvoZPkTQ?+LeR%zueL>-C1Wl8=ZB8K?dz8 zxahJcM>KmHna#Ym19Ce_PYYi=ojW%#PD{uhGAB=$B<{1coT<%=pAM6Sqg?jP5})u> zhFHJoH2>rgIfR>)h;Rq;7jGJ99Ox`cJj@>rH<+5}zSYjj`o31pRb5~6j@H{l_FvHg zwV$Njv1i6X=l|Qw-Wat0Nz+|{(_2+(eM|21!V=1iyzx}LKT>CXn*A}(T1#QqPr+Sj z*^2`>qCTI!%%h!1mEcLugselB6=H=McQ5VQNdk_&51Gz4cgpV0e15m>3 zm9$Z048K(@yPUnBlzW4!_Pp#-YdqcB`XNB7d3pG1Eeic0d4s{xBc4;>TN3SMB6+Q* zeSQC}-!OmrC#e=3E2l6PXN>#tFrGk0h28awGwoUDCA zvCl8Ym-CZB9w}9JRIFFj7YQc>m~8PW8nl0%8x^>`sj5|>XWa`T#xnTAYk%4vnulmH zMee7@9=A6<21>py1b;#cz3{}pye^7n*ZW9(ootlwG~miNAqNe4S#rf}PlZ`ykE}TW zH#)r>&%@XW87n4Sn3CA_W!ucell^09qNmxo6UZi**=0}hIs<>)AnNw|H{!Jbb#q>h z6wwAaMf?i?He$ls#D$Kr3@m2M_921EtP(q(VV|3=6*9t-CCK^&ZyOM4i0VseenKa6 z^$i?Q3Nq>znlYK=3H{n&a8B#q_vB^F;~V%T)U6VG9@aDA#Kdpr|7n!*VMZ-R_DMph z8c|_hIvzPa)~9O_M_=#$)00cu{ii=1kbU%k&4p<8u4MJlPih$9UfJagB=EkfN!e(w zgDzB+6)u58#C^uU4EkuscLN6;KV=(urI_?%%h~dK9imL4UWn4sDah9>``vac@4|L6yXl;8GR+ zi``3~Lg!D{+u%&!Jal5K9+I{B9=}CSd4%jak4wAh4_{<9NX!Pl#FlHU0FuV#3oY4CFfNxHMNDB1UaBr=q$n@^)U2e zg_Laapv^{g*6hS|cS=~>mfCf5(clo%UWry&9<}U;*mj(4H7Dd(L9;pmc02VWCs%se z0wgI(2>kL&W0aIh6yyt0C|^b`_!`6a#;S}96M0U^*WOHy^HDU zjmvhUwB+0&xdzc8sqm%MA^P&MRYjFfn2NBik3y2zM9U0gdM@cAmNUk7^&-m6{ZmYU z7Q5i90d5C}`#KCk%~30V1wqRBkg}4#qM_@Kz?j zM?-vg8QnpIqVO&FC<9T#WepMrDauD_@W?NuZdQ~VEqqU2NT>>F)wW|z{=9XcxovGU z7}FS$E$ETSh=m795OEdzdX8x}on0X!g&Zu>be;M8BS0s{AcCyd(_(t40J`iXRn)lQ zdH%dqp)unMn!g4n12~uu#m8v+>nzF`8N=szyztUuB+~TqIqD(33^E8X^jS$54Cwmv z)yuF*llqZs`Fyp|`pd$%!Xpe8$W&y&OoXq0L4E`-Sr}25b4}Q{A$!}r?47N(`zZb2 zZEr}TZadN+|I9r~klAjP)g^BBdRx?oy_Ek|f8VK)^{%Xa+aRBKY;8XGZn-~Dxif^i zdf0R|>}ZXRidqNo!!C*NxB-lCx`7ZL#D|Sbx`IX!JJktncnvioJgSkw6x!CrhSgV=iw9rlPf z?MN(+hgme~DEIZz9U5fMdLRbpNZzC3Q{(y;-yD(wm4xPJ(${aqfy!q0^!`uVKSwHy zIURgNGKcub@^%lyJU@Y+wy@P86^80CgB4ooESAWIm18Zla$;)99@084$i^%SjelnQ ztZPU)TpGO2R}(mvgSdYST@WAFWK0o5NY-yuZis&ttVfAD&#D_ocGZd=9@8r!jWg8R ztBIe??}V1*>Zc7@xFHVxv!n}!NlLcO6F##5^HlflE+bdXcw*OUU5qkJ^T#<65PMOB zD;tlC!QAhjpYD}JS}vNby%Gcidz)>exHUY2`rtu++-|#?=1Q@-SXOM->812lq>CT8 zvc`MaK@RJI5T2gml>x-ISRA8|_@0!m31$IH`Y62fM9=lK+mBay-Yo!K<4-ZkWXZ3l zdB}dP`qb@rg|-vrL<>z3n$Gb<*NS5`95ME#A(0W_$dI6V)8sS|0RlrW30G^b3GXES zZMx`_M~Hel0TWtHr-ccDHN1`LY0eeK2}j(P%VcBBT6%6%L&VrF*ln3)4w_OD3b?z+ zr`sux+uwFMvyzN)WUCc)Omg|vCJ6=(4l!O;aSWTt$#}Uo-qih`p$bVy*?Srq7bhIX zb&1zKbRGn5H^P#8bBYh4$t2ja%Y&rvsH1tOFaZx=;~pEoghbj?lviPTLWr5vf~NOJ z*O@bT1_%k+5lAH2I&m_~N_Q2q&-USIbDi7YbdawH%Fc93qq%zHF=vq8nIa8iQ!yag zHYuyiT=p7-#F2RpQZ1+R8lIYwiaW+Zh3ox-FzbR2nMnO+$3?QH*?P7Qdptrk%|2iA zy4c#89a>?{f^sbq>TL#=Un>O?;%Pi5Kn(M(EI>_}^Lg!*k0J%L*wv9_HHAoRI(U8u z4PPtl&$P4?K#1ulS_^0lBT>ATT(A3p^cAbNBHQ~#nH`;XJX=8vv{OsT#Kg^7rbjR5>QKYW z`VNPX^VNvp*c)Rb*w`J19+v+186Y-(SH8pWeO-9}(8#)&CD zF2xM)34)l5fBLejplpH&Xk3KR<0% zc8er?h~0X$YREqX^!2`h0wwHezY*(Y3TwUcMy;B?d!8=LZbLY3@}(tJp%=a&kH%s& zZv?T)-qRPF!P~&syCXXsjtc(4t^GB#og1k(mkN#lrTpep<7`EOJl&HtU>uo%Kmp=6 z%AhUI{IsozfC%pQzD$A;zg8x$t*o4o$PmOZYNo6*6mg$#Lo){s39@cy9gpqGjSXw~ zE1)S3!Tr%w=Uy3wnX)CNR_R>Ici-DhihFlK1M8(gU{Dcrf*j>ShoEE$9P5H~DB?Q3 zptRYv>_Lc=U!WZj2Y7AyRQ!1eC9U;x$S$>JM&Og{({O#4~|Ah2l>&`3s zGEQi}0~R0P`&9|_6RAN?(C1_B#<`$M2_VO@+fBmnu5{aINMFJAX zN$if$gC?876;br@HG~~xlaD(sdr$3)I&Qp{uQ1T~wS_FdA|*=W)?3xwejyA%77W24 zBZcG=;av@y5HTjvINC=}wXwJ0VqSUR z$U%FNEdpEl<5>VlfOd|6-@94>vOd*Jr|5e@5Sq~!0_H;|7y?5g{_nA%G{Jvsg2ABv zdmkCCt;-K03IW?vG}3|TpUee@+aeB}=Xy#=Ey1ztvZaM%Nor`l}9G97n)qh&w{FFhpvg0G%vwb-) zT1@}f~$NO0)u=8_P#p3qnED?<)m3$rNYYUaNmwV9cOz@Fu z;76@V9f%sy4CZq*9&3fArEr6`3^H*d0QBkQd&<=}7?u(C2N>uUE~^2ESshY)&Qg zCae@fA*Q0In$D_GD__iD`1Kj+%V!vf-`JK&=G?|UO8#N{`L6q5Oe*SKOctC58}a|H zg^$ZTW#D+R*38O^36tmWJ^Jnb=Sr%2p-O@IywZr1}K1A z`iIh2*ed2Dv~{~Xr>L@Qd(rf$2Ja_<;sKNV)|h`YG?|17Az+h-c463@{k8G#5&4gc zOmL*?ty<$`ehDQ@DQv9<%ju%LetV_8h5x{m6<~arA+dZ)ZEZ^dBxQzcu>L@prR&1Q zG-`0nDNgiC96$Zvcj-Oon+P_{KJg)0Q(gUWq!WULFiWrOU>XSvcgj;D`v6=YpywLy zzwkI&im_n$^FP=nmhpr5X9Rr^s3iZOO-L3^An?DVyx@kXPXHMi0vV(vCyPR^HPoeI z;Oc8&HN5J#WR_lVFnswh8S}Fj;j_dab!L>CEyu236l&N%^?L8PJhQ5M%^m3D9+0#r zx#I0m&of@fX~BE1JK13$Q>x$?-ye%5A65oS?;*OqZV1@1^9r8*T3P2`Qd!dYQouYVdv16DiD|NN1lY^RxdPKm>)Qz=E$M zpi@5Q!I^v`zNWXAz=84_&Ux||qqMvY@z5u&W-D54OTIiY$QHWy`_jj4A*qVA!HRDC zva9Y-SQ1`()NlX>*^e=bq`!mG8>3r}BxfEL>lz6X9EdT6o9c(wuILD+BRjXXFtqAay?GJWqRXWQs z`(V-PLa$I zeR9fC4EVvAD_2(1$b#}%s3Hv)Uuk?%U8L2SE?5|ZQgi+oU#$lb17Gbt)@O)$606O_ z5+5ye(xFGQG{DjFiMB|epCjy1{L8$&yh(62Jc&$rLT?7#2khu!W*bNeIhV3{;mji= zqbHV@OZUgZ!7B!NkEEXTQ|JXosI3(* zAMD#w^y^wZ0(nl}@z>O#%8bi642#zu1<|mOC(cJ5o5KZ`Qef2#nI3_oM(oCvsb5t) zcm+}so8;>~!m^3eyAu5vhMigBJlD9wP|}A!jfz+PnNAcPbbFbPkp0hA{c73}5o|mF+wLmJ2MkdTBOhwoif`eAo`7y50abey z{F_-Q>R|}Z2Ft%0yB3co_|OVS(*295iT=-OdHDZ-BF_JK>jxA}phtwc73Jj>?rMze z2q1G$@70XF-;p@pYCo^We3NF>i%{yo#Gi-XBuD(h-IIqgxIyM-pk0kP_@{~`{@w=Y z$Pw&4S^~Ek`&n+wB;WmRr2oxPdORL(zKbXMpLV9D#&<_l(0IVT#f0a)Vy&ml>-ON8 zR2V^v2u7viQ%Xf5G!i4T36MZux4`bo$VYiHq8|tY3cEpI)cn_30N*T&tq3<4A>!{= z7clMhF!j6Z2pm~He{n=u32e05I;y(_&BC$*!P*3%95Spv)DGScF+XkALlgWN?s^NjU2LlT}y?upyKxm z{~H~}><%fDIM&l?LcWL}k@J8kOCz#719X{p+Q3ZFo;*m4*Or#n!(-B};;BP!@`48cW* zYXCNDxB>RGaUX-MWxu%G5hT~UZwWcTA(0kPQuEXd-`Vx8pV>UiG7FfWzMkjP-VbT;LxD7(5Q3 z7ym~1RYAe=;OBK)X;UnU#&bsrkK46p2$=yrE6aPs`T18zxh&;;8S%%`F6n|4NMcQ}!C1Y>@dP^|tA>q+KiWzWv!4|-sj z5#K+je(zTly~`%r5PjjK7pa#%x&auXB*T5JLzfK-jE(m^j?F!^^?=XvLA2JONcD>* zO8I4xB3&zcF4UkR2Q@3px^0h`13F&pJg0sJ)6yyHdjZ?Hp29YnR5MHj^1GT_MWVg% zKB0$P0~!+Ag%nOn%(LT)oX1FgQBk$ArDTqcyN7bdb^eAOh4e>`b$^=E3x{yM=RGCx z8&ae%tjSjCy@d>QBGwQq5FqpE&9XnQ{A^bh#>N+@tulozPO6jJ&<`P;iZ+-*6e^16 zrRHMuO%Bnivj38y!SK6a77Oo_qs;B(?7_&{&^E|@5|^rtG`KYno;~-_ki4H3jkM`!7U_^Foc^go$705Ray@IRB_U<_QHA){wTS4u zJyTAR{v+yjwf!0Odpj9~e~60-Q%cklWrgD3SbUcGZ-o$k+WCLo!2O?!MxZ4TB@buT zTbq>r&>-J*ghvem#7kg|rt8%YF5fW{9h+8nDgcdM+}>6U58>tT=OW2HuM@n7b+GP4 z8iuZkg-e=8uQO)A)O+d~Pl$P{-9j(H?<;F2y>+t7JyX5EBUCcShk+BkU)R~47E0oI zPli6ny8}1}>aFqvhTXVWJk?P0vf9#LsA2q|B(;7}3uXJCkUG+-p(fmk0vVYGIKW|yV5O;rEsxPaqzK!yfmy~L{a zRfA{UZyl{`d$StUO@jA!<&){KHv=IvPnUY`@%UESG?N;$;;_%}4J@7#D2iWu*JkB* zeWGMCA$)U0udGjogbvZ4rz85zmoPi(BaR{A;{Yc+IhGMU2hW$<2(xxoh8=!%sr5?o zW&5t@6!*P^H4~pHn@suo(`8c*MnxhZwA4E`q{jXjzt==Ql_KM>&jqRu#kD%afYxik z2Ke3ZQk%O*QSq~>6)=792mAlcEkhi>kzZ5SDz~+XSE4RU^J1>{!1p4R<`eOE#AIi} z+Yfkh)8MUjltsJBAM=H_)jTOBk11>qS&6EzD&Y578D!ebg5%2E(4S;su24RU81mKo z)L)@ESKDtulIV-{CSiqku1eY3!axz%ueccs{h|V$hXn8Nwz+~^6{c?!VFmhuUpmZR z6trY#7FN}>{8$15t*Fg*EQi#3A|s91hT0QSxvE34Ii_{41){5l6ZdMh{Otz&L=<*N z!BAd&m0XJ>x%j=S@~Vfk<@TATjfJbhw@+BX zLg}aglLi!uU5CectB1opsZ5}ep>wa?q%U$Loj1U#tTDT&H4Jpor23mAl8Y@@@5who zE)sT59P55IxMOew#B92Nm2;-c-YdbIZI42C_b&v@2k1>|_3Ka%SU4LXxX<>d_la5N z=$+1R$4@?qjfrqC1~%!XQpyDH0!yx(I3z)rLwCXR}3f2*Irpa#3 z)QRM_1sUgUc&j)T6(<9m3^G`ho^{7kNSF}fX*f0be5vOUCM?3+fwe?UVKgDZc&Z4W zx{fzMG@>U&r}~L7Te{&e-op?5W|spW?72~8DtRHNDnrDkX8e-7LFMredLRKW-I^K% z>9Yr*g<)k%vrsvgKHIolF|UMIWVN@&gPT*S_kzyQcgF-@F@mI|9POXjN@++Wh0VYD zIPO11ZmdRaB?OYV>T&wjL!2oAk3Dj1Mev9&iy^DX@FForskUN`pd zHWbMCzp#eSmFhRE*>&GP%cJ)?6UbaAM^z0Se?8fp7BK{C2%$#nI*;nol+mWZoiyq$7iIx)-EGhJeH=jAY3N7{(Ib;X-yb zgOp)+60ybuT5Lc;RD0ZkB^K_GuAk^xN$npedK1I>=W)1~qz*8`!8d5^?86Z8-hfvL z$b5|8an(~Zt^$db>6`60$Gs=0o$I zBZqBW5lgu;W*-wT)wNSn%D?D1xzqF+8i2VqM&>>$-@rk1-FJawDkEZ52nIvoo>|@w z1wsjtR)fzBo-u9xYVY;r`5SRYqa1cJ4o#p1jq`h9vducRyt+`Z!2{r(>5-i?PyUYb zQUJbw=(puiqSOe%#(?5`Sg`}_Ngss0=#XGG6t5t}t5D)l z!6xwf@%}RToe{Nv%BT^(S|BNGtTQ<<-M?D}pbA!6ISo{u0t@t4?H`;{XEWTBrn#0F zA3SSRRyD+289+g-i$<0}aQg!-Az=;3J>EPhLygE5ByRL@f41@Vvhi<5=blQFFJ5eV zMV3_!OAB1aPFo+-{X3tTNe<-y!ZR=>`B?w1tN$~c_`l7a@c#+BaSwoo6AbDlo;#?s z?K&)$no8hq5o8Vot7;5px7j&m+Wa-J&wU9KISX2P*rTtvrE2?*^}O{N6k3zD3}Wjz z*tV(@CvTgyLI zRA3HezH5H+)Acq=j}KXK`)LbuA{yTKCbfa%b(v|Nk)Iq+(?I|#gs{7E@PPqpV?Ve< zYOJ<-Z9WpN%WZsfQ(5J_D}qkKKhIXUoRzC&8J!ebl?By^C7wb)G_5_-3^xZXYk9We zD>mLiJf3^Hw=%ecy45h$ZB~6iGk#e>)&rJgePTOJyo;rdF&_&QAM}T#6(ON}u!BmeBcJ;`t-P>@Ae0C&KNBNg=#C)vrS=ti zwWdSGPnCR&0S`?Aru_FMQ9kqwYf>BLt&UkMz59LY$&6O5^bfI7Ga6lt&&p+6HjG=p z7&CzI9eh4naws}Kt3R|w4fY$pT}t-$%$2S0pIcTP)FN~zEK!L4I_mqx3XZ(k^d|WM z$4bOx_XmPMH2H;VMxci=nKnd^rU;V=h^f4GQ1$GA!}hKTuMa`d-tAGtnP2#6l+Iri zPkVtDByLKv!wNdMZ-DLQ^d3T`M z6T00&B&wt>m417v;CBpZ(h^z`*;?NYC=1;Y*Ix*Y%XpeuJ$v2t?ZCM&8RG60*IQ96 zKA+6@j2AGjT9E>6hv9bQCRiOT&PXcblRqEn^4~S@{;8(*aU|I5SQc6M(RfDX=Cbk6 z+p8gLzG>xo&_gL;3qfcw5Lkh!kq$ikQh^)sitt%B**1^uvaf&l#TMr=3sGP&%!TAx ziErrL_4W%CobkQCSZ0`&jzEp(Y+eZR6$}Zw;5><|>Vq*T;^jx!kgv@_zy{4&7k^32 zpY0%`aEYrEfa=i4Cep`|eF{U3_K8gTu2L^_d_{bWN#)@b(&VDnz#(M|fh4cZlDT)| ztcS6Kr4$JmCH7NdLY^z#^kTgo<5x-Pa3C~-g_Tt-O4N%<)dbRlNd#y5s>9dB z{PU}mjpRqY&Iy$p3U9Nd?)H-r=fa+hm7MEWKKjuwOrox0aaAs!!IDlU_|BDD1UnC& zNW)zEodpBS?C9n4@^L&@qxsIWb&?<8)(*AvMQ$CVXzv`EY}JUdo(IIfD)db}B2^y7 zLH@UVeISG+>@apeb0vjan8I#Z(NA7kia*F~FZ&hl!VmEp`AYCNvtNF<1Ige~R`T`c zB6CZ4?rrn%(-eLv#>Ju758^paZ8Ri4HF9pQE{Jhz_Poub`s##^awntfErr&6@BTu= z_Z9&z!m9KZsaT4Vyx1x5BlW zT)1lrjEzpKHNxBostF)xf z71IwFl*pRpi=JwccKyCXkiI->+cOh(rRJ9gCF?o z+CnnK^TjVsFs(oPpLFr|W>Y!7Pl{9~Zgp*wX=X!B5oXEg>(OTk7D1>wg1lJkM6GtLfwxXDJE@yPL;SGGIkiQ|!8G9JNcf(R3+iI}t?6R4cD9}I#sBe_ z1V=}i>2w9;MA1WxOQOAtiei?;NYR18Mt$ZfwV2dj@ z5{Yf^+nwHN&Xx7~E*{5s78nJ(xc6zUQi9TAW_y~Ow{PklE+qXz>IQPL*VmqGS>L6b zDEN;&Kn?e7FXzM%0*coOauMTdHNGnnZ+t;43?kRl#^IyN)QnQAnZ3K6*SPgQym1J> z5(J4}zNfw%oKv-SQ-Q`fDY0qQE1Lm5%#5VCoiOS{eswVRUxCY6)+r=Y3IE;@IZL11 zL*waSeo*|{5&8x+J&ym#g2Bc0SR(k$;=>`3nc{64G#0?;9=>HHdDRtWmv zbg}NDk?{prW3|WP+nX?2%Op3Ug9W{3Dfdi-mFZO@_h)@iKG57{sO<)f(gau<;wg{hATkjLXB_}V3mB^<1Sy;bwsA(F&8R;dY6^e6Q zYlq}D!i}R}Tg0}#jpzZRC@a`UwO@~#Nh<*95WH4IS-P8JAfm9nbStj5DWO7%cRsL9 z(}>gTi=t+y(aSFT{|rr&&*cP=EFxkkXecL;Vd|#ewkg4cF-mK-IMbySPR+~&T|8T) z4YYaOq;T64rxnH(2TioGijeEtioE>6`GubM)721Tfo>w-_;Z zI|NUDi*fITCgnWZ@6hb$IpawaOf=u@A!^#NEV!JqyS>@R$P?aD1E{p0@#HFbtj~*r z*H@C)O9%82p^?i)J}?me&|i+ZIx@CQxnS`;58BnNB4bO-yVeXE&{6DtKjAv{5!@=G z9(1DJsK&70BTVZ|$$ohpr?Z}{E98lAV;nfD`B+I}R};O(G@k0N&wZ7=z`o(d#HE6M z@6)j7AwNIdcOoYhS*C!GEzuro&dq)v5{lZJH%$)7=Yfn}okK4TQENcnQ5)eajw?pH z8Ce|rK3d<%PZDysEo^c(MlpLP<+XF8(5I4p^x4L*fBbCthF5r{L+LS@H$)##=lW2Z zf;`o$R-`*7rB@h<0L(5<93yg>b~o8TOFV91>MMH?FhQPu;aD4zNXBdUlcz0%iXqlJe+vay|tmZD8xWs-a3^?-#^HsS4{F-nv8j@W~vXJs7G>UMEPI{2`{%bhZQT;pY9hB z=V@PyVa|d^iuKb)loWLx!eX;zu2(5Vn3UcSbc);VFp>FJz}L^o=<6lTR(~gcNKWJZ zszab8WZ$i)9@054FNsTEvAc;{9rj0aAK5MGKUn3fB&zub$#{=Gsi7BH(W!WR@yjhl ziNx38KyH+ab018*Q;HiD-lPiT8irU3$M1*|Dwnn2hEdeRz2CG{Y5G*^=-``r^nP!+ z4ZR$5+~x=%>Vi;*TTe>7owz}<=yUowbVLiX$F&Hs0(W2xz0uz}gttRy!b^J*+wVnW z5i~dL=yRaKn9eB*YP9FO4NKP;Io5t--UHIBLH<*$S37mTFz+^au^Y}Qe~i>b>y!CQ zFeV!g*qi2l%(-ffj187j>1 zjgZaWMjR5~w=F)7y-`R7(nhnIxH7)^+`SHH7ugttZin*6XM*Z(WM=Z*D@8Qt5$_Xh z0=v>-_7%*h-V28Kw|*n!a^A&>g4-CF;=0kIyI=c*$)T~96drcb1{_LT!)NO2&WPRP zWaZKbww>cFKd$V`MJc-1(T062&DPt(PqEh2pGRoj+;)gK!}jIk6Fc8nn)#+VbQ|DJ zvoZE^sVHfXkU=*N44>{aNnil3aqr)lpM(f7?dUNC>;=1j2}Zs*6YSg&;lg;EewJkR z3!Z!|D;9Enhvo>r&=S{ogBU_VVtoj)BY$TF{ISLsb%Op8w9B+8zuVxN5%Wr0B@{b< z{|om)Txi@{Bl6;u+I!71KXUDMt^DWr?SyA>oZ)oyglH{H<`wzJq!GuCCW4#o9n0@} z#<%sE3B;+T&=0*DZ9rW_-){12VkDs%X$u!f1?~Y{ekXr0WzkPZBbp-iG|L5YGIao~ z6wp_kM%df zFEDP#B;Bcl(@&N3K{&T+UTWdPFs=r^H%s!pvalF*vgP#Yr8WL9T?;Cd>-PeM& z#@H=d>4yFkxf*K{Tz1rATV%WW~$8ZWtyzkL|`{7b2 zm296AJR?88Vy&7$?7OSmF`_=tUIo{q@14VUB#3$q<4}S?#>#=VTC$Kwjodj9A{F79 zPBL#RSK@)~CVq8cmj$&$-ZxN@Stos7ZRz<@ubV7eZbV8uVNyr$>dfau?6Bqd^Aioq zMAtHYmT)=6pAJZZ1f2->)?d#!h|df4+AzLOJ~1SBqo5dcvWMP95FO%lMh~Hjz$-76FleCaZs`;vJ-(Cg{^q!#O~@-O1bV+Lii{qupy$Kzc>9Pg)-nqp_^iO6^t22We9ONdgql|pw&{fVDd;X)# z7g$3Zdl5VLU|oT?UG%kP-i6gWLFQDSFMKqLu(~3i!%zGA%O^!Xa(MwZvaxi zia_(SHhVF&j(?GzL(|KIDlPVce^96OF=d#0`XqW~&?Y%SdCh!h;dG;lO#?BPD`EgGwER0-d$3{@pH$WsBGx0dbd zhw`Y8#rh$^=QkZDF!;&133z%Cz#bm;G%7=9;RtUiC8GFnNl?Wi2mNY(Xhe*AKDL`p zsx9^s+o3#JQm6bKAkeITM(PP->tQc)XCW@% zu9U}6faE~cKHEwz9EWqVW#`S#YR?D~h$p5My&WyANoyj10I6rP%ahX-OdTU)YW6cf zO@fKI2^Bo!TXs}Mlw45b@8?QJHyZ|uV0_Yr;Ec)rNIzFd7Er;00xpw&7T=msyqnpI z|Az};C0;qqZ}~0e_-=*nHZ+{0l^<s%zITy|3u zOXgdl>f{&c0jU)#nOIb7g}vRpd${CjJ*Fr-PfSk)`-CPNtVZUX!n0CDpeq3++Xj)dWF5S>gjhd$3R#Ra>pN)rSDV7WXDO%Xy)k)3>NKJ z*uEwUVFu#onmdI^Ne(cLhacT65+_%dOPTt;_vMZq_{JY}^lb5~Ue(1LMi(Q)s zM0b;ra1#W+^&!p@X_RYc{IPL(61L`M0b$aWkL>VI%RMsFd+y{DSq-#koEC2<+pex@ zyaV*|TKzPlk9PCJ zbD2Y|1A_DSi>IQA0Uj&{8&CzxLjH`X7Ed&7Pj|1M8UWV=^)9vmDAO}$E(Vybbhq?< z`{HC~xr6+H9tz?UHe^p}X3;Wvhp79O)b%i7jxDl0?brcAGFN3~#V(&_slqd+L~rZI zf>#jRM50JjN$cnpl*?jqR^eqwyFJRoYl{NypJ`)MX7dA@3quaebYPhya;baV@1vE^ z@X7sLCK4IHh-}6pNslgEdeS#7U3FLRx>HbPFjB4dsJC|ttOD82g>7vNP7PwfP(&Sb ziyG0rz6CFbzF_(c>RRbA*6rZ4?&CkeIJ^6?k=W})oA2Q;*SHTdQD7KpN{&)FdpkoP0h#(}3ug%36~? z-Q^NxCp;ZfL;PVAnfFqVJUxJ3|Lmr+k}B>~==NJ3%6JF$5s50}qx9N_=PuCE(jj<> zMO@p143YR(;{53QN*4(Rs+wi=InXw7%4@)*KCs`)AfyhBDSbgeq0+z~{Y4p4mtOW~H>>vaY{|ET&QVx=-P z8A2X>xNUOdR9_s^6ibrkZzjvm5;Leg^#f^!>y-T$)CP{|can;R(|JdelWX4D0&8~+ zOw)qe_lEBM>Bi0bd&p!@vveXle2^phS-lqq3CcImYc8Fz$BCn4lmd3Pm+0fzKR&kP z8cI0k$<{~pFU=NgmT%4TH%c-1)bvVag|bKcT5`VxaDsbmYL(;RL;j`DWaoxwUl-cg z7|6%#vYAb3lwYZZ*tsF$D2KLjbqUgNc(!TFp|{(>ganmw#gdSN0Jh>A^CDIf+gIR| z=VyTGi*|rP4F5_{I!m_LL2wF~8txu?xF z=;>fR{xdKMtn)N)EYf(Ez#ovM#`3jb^76_Fe;^`8TnS#~bs|&iMABgxKgSpyMzSIj zJ`_6D%;=211C7d4m3=CSdY&0%w5HbWeSsc7(xdD|M&zfO436?#*`DZ|Az_Q?z{gl; zd;-fDxxKn3Rk1?zQkok4Yk-vvMvqXIQhbur|E>|@Db2Mf3cXl1U|ZAG$o7wz6Ge@m zjY!sR3%q(vmm4f|dO*%sWxh1ni3%Eyg@UJqb}f|M%Wbq#k2sm9$ESxcSjHOH5iimGt7yZlMC3JpLaxe#dWl(LdLGdz1GaGPc=j56HS-5>Ypx?78v`7k)wz+vhTl)K8`dQ7N;eCPf)6BxqjK3q z@BCmKMGeWLsG}yaqOssIvOvo1PPnbP=mLev^;1QlFhN2cn!37xNjgh)2I0)~F6pud z=*eUnWuHm20akpMmv}$}rNV6r=9?I#`g@AlH-V_zvJ(|fNFQL}YiGMwCf8FBrKeH+ z&Ca~TIbIH^Q6+^lVb)8B72WZ6QA<Jf4z zK5~lx_z{V_O2gB3!JK4n3?Rx})_HHtKzMoaAU|D5r65&EWMD4eLk?7gsf;%Y`JmY) z_nlgs8B{&qDkJvsHKrtaG^5g>ys20|ZlI1{Zia@??3{w4KRKyFt*5kJx61`+=fR8Q8{A-@E zsdj5XmKtl;50uFntW*8loiS;Q)FkpcUT}mH*;3LI6%{SC-=5phKm@ zl)U(2n(oP~R@sZ&!vb95T3h`)%&52eWunb|PN~q(B8o~d)u!+j+N5BL*E2~*s;FbK zi;aOlg|GM_Ci?i{Ws_ZYTDs23?RWy4S>!L>eLZ#_v83pracLS8_`l`7M2XB6I9ds< zgw)T8(CN%xA8SQv65sB`AL2iJVTqO5v^^bc=?wnxM)Y3eT1_4!pUy{1ENxN@zpOdJ zIRv{DlY!!tY_BXGpTz`9JPw0O8TO+gB)Fu%PuH^uW^khG#g(onlR7$|-pRSCw=5+UZ* zxYiUDKLX)vs}5ST$oP5F)qK#4V>w_9_wWSlDdzMh<*}+5<$DVl;5a*PE5-iduWWz8 zD~plRPzGuT(dk_Vl7g-HwY3Hq6|13u;e^QuE8mc0-HT3PCdzEIQ5AyjcemVA_dgkL zAZUoNCdzpFv@JqI#VS>I4knalr!kM6DF5#~sWK$WLKXGJ#aVR6(cb z_5AMhBdU*K+7SWWW~H#booM6!z6dY0JL8L!a$=lDa_c?jo6@Gs`@WPP_@NdwSsB|wBR)`rg zXAWl-b9+47h6Pm_4QjG#4VNVA8}QvkRDnZJL;LnGqpRdh8gO_bZ1hoH8Egt(eLZjwJuyX*)@-_3i1TN(cyDS}My!D5){`Izb-&7BXDi z10+k<)7a~=B*W&!8!ewbly0tThOYu*8x2ur(yJNE#tbjJhM6%K7-lN=C`XjM z<|=>7sd0#p*mf_!xGJ_nyc$;%?c7D#Y1`^eUP*|iO>;1m|xHV-Wh-Rkem7sUhd1@Vtp z=-!_|3ZAb2!ld-9vXV8@e!sW)^9sP?C#>JtUt~|P|2xanlAx9WPy|}6!$IkEsWG@| z4MT9&V(u7_>T{ucIiw4HTkOBYS%_QnHV7at5E2wKn?WcG&N??eOsH6VR{N1(wcg`I zc#a0^;C9D$`;5$gqI?k`cN8m-f1z}xiVX|*J^j=+ovv;P3Mh88o_VYH0#>4M{%0eT zEhNh>{~$N%w!@qv1R1BZIR+^Ab=ql2wuM={yQ`wz-8S>K<8CVM7J;Pj9cIm4%cRoF zhVrbw>`xt!Kj3+m*pvC2%zbNC{v!9Zl}L{K77f8+hY6L>`%<9SpJiUmi$&QBQ-&ko zvEJKPZ-23>SR=KEAtpA0A5FHei0joM)BZl;-hv=w#@mh~v=(MPnGF{&+Jo3zZy0-n z@SL)@f%O@ZA`6o0OlxDId8y_JzHOhgs>k+;$@ekr=?%~5oVgI3w)TI-Jpsc zoH#MDtQyapL!U05Z1sKcT};rsjmIR@&lJ0EWyqLprvD%=ztoQvY(f? z6qWaVkt6M^^WxI+)ae7J$`|`W8H%6dk38UYjXI~7oa0$Ws7T&ijmyEu3j1FPGqqSQ zZCM~+y3s`_+l1CGE~xISMfO&triaUxX)kU?G*~uxGac$fT++w6}vRXuy8Qv9=}A-oZ?)hIfsa_JQPiWg3o#H#(q#oi<$s`u^}zD`qbXZ!3jY zpfj$(@3N~=DWr<{T-I-Y64Zh}icch@mc6=zo|Ys&ht<}ATwal+-Z?=jZSF_}d7}>5 z$-hiZL}c>SLjB$MOSjL7j86sLj+axa!#hV;!9$dS){L>m3_8b~CEKRa>&A!2i6cjk zy7!TWI@o&47-G4i8fzYems|MpbMnr{BUDKzGDdspuk9*ejuv}N=<~@#D|8=<0jke| zXzoSu^{}pqCkm0QurQ1Iz(f*l<;6go19cAX&E%IT;wQcXWsh%EOI5ao)g`f$U(klF zpXn}stE8ce|5`wE24Sdc7|$O)G?VYObLHS^%&xX1ZgHQjlN3XnMs4edO^pL3P7#f7 zy%|(n*u_wpc90lSaFi%lEjH_smA!eSz-a3Yd#WtK2B~2OVR@raqlmZg{t-@u_u>sw zz#;p27x$skKRKl8EPr>N+nPYZIzm#9uk1x3p`FzG(R7eQp4lNAa&S2V1J9BC{oJof zII2t)?JI0dL(jS{bVozCBtqHk`R8iSF7p#LD&Bxdhpx0FiDhsNHR(_ddiAVUP3*-p zcmDe3Kc(%0ScW0R$nYN+J+^g=cf4Uza7p&7kM|`ZCYBLd*5FqiXF#FpZ&SObDq%Ue zPc4jZ^ZUQ;jU)@(|7gx9cPlGnb=|o;=*G>xN?D57YEXF(ecqO{mVUVr&5e0}+cXm3 zkG}rp338!HDh|*#%w85V>6nWU3wTk;%ySuwYb_zrTu?^Gk1WYi$1Q$dhx)Bmc2lek zTxD)`po{CYSAFU6XTJ5%F%{;?xhHtPBOBU2c@0&sOa72C+Sb zhTaFLIDxs@)coZuyfr`aDB$!$XU!Xxj~&Q3h0lS8{A?y2^O1Q3w66jtS+u#u*;%YX zkqZkL&w~?=`9n=5kxKw4i=$DtX zJse4mDf7AL3>J*6cD0jxGg_oFO6G7tArB7&pUj4uq}uSq6fHQ+#?lvdktg}x1ToE_ui3GN6 zkP^8_Dyss$iB5RsQZ%`oX|(k2I$pGrCl)C_42A9p|X%zY)^Ze-j{Pi#Qs6O4Mo>>k$!XI&B9Ba zPpnnn6tDGvBl3v`*%As;$L0+#h3Jq`bh1k>I8^A1^$PoVUwmZx` zOY8*pzWWEPs0?J_ilF4CAk7`hAjNA$bLhTg`yhODJ=-A!ofL2q4)N7_BWCT^*B4O= zESR~y!~&+-yVZOjL~&Zm8Dd>*gDKtY4^mK-`-|4Ig2YRVq^ z1wHNVS53{U%P@v7d`CBju3?Rga@VbmQq9GlyOn-zgg;#%Lt%7@FK()prR8CrW?IYK zoo2T}!w_r~NFLkH#MD?gpg;ISn5K2?1=SAmL-jWt=gX?p?aXB-H@gk!I;v9tIJBM; zL86ujZwN6`DZDYmYjo->-rZq<+#Vn%x|svp9Z?rn7LdU^&Q7y(*+nH-KYk9Aghf3u z7Glo^>?@i%OmQA-m{Ba0cw|YQ6?`JwF7ai!p1J`3Pr5@V8Wpz@HlrM}(9ZU#ADxi2 z6f5WUV#>0_ibWEeE$&2A4jPy`-P+&8{iGPX)m}GCHk0Wp$I!L4qpk@x0fMc82CYil zBxboLVOJxpv(o-6T+)_2@~3r+Z*6)?k*Z;`FQdC}91?0O37@Po;UKvJ5XK9U+3rmv z^aYZYlfZLOmrrG*OFmh#1#0hY0me0*xa#8G)PyRb4b<#Lc9MzOfNVSyE=y7oKE|vA zg^E7_dps|AWoRv);4T}fF^w%jMqYS?B#pg2AAdhmNghy;3x&Uz1byo^i(zw0Mr+MX z>!87bC7~%P=?_6#l|8C_H4iep`>OUha7?k0`cG1cU3?&tndB4&4wkx3I$BJ4d4o97 zDlO}puMg<9MY^7uc$Rynu7?P~h@I-xbEH z&yC^YH&}&k>>4_|$DooNGD)5!9DganRl;2%&c_{$kC$nNQ8`e)(LY%rmU59UxAsg< zJLO}WAKg}2o_jp@daGams~lz-tG3t<;=t0y#+)6NR%l+%VVtt(`FR7mfX_AIaS1E{ z<=C1gPXn)Mz+w=ClwPBtp2JK7GIk~+EhBcL zHhBx>v~Sr`#e2as129feQ#X3q+F*D_m-Aht=lPN!J(+d_h!V+m9UVa|>WGj?;-rGx zr}$0X*AY4A0Rgx0pEj-UMm2DAveW7M_k#ZBou6y-bvY^-vw;+#S4R)a-12V3&7sfn zJMqf?w3~2R+N}Ga^G@*G5tYn++!H(H#>OQf$`~=t9(u_0lgfk*CEFEh=5k)&(tJcc zx8mt`sUtrkjGlhlb)K;l{Hz40_yHgjfB)b{>%MScSFKPJm2c+@M2vAsUTQlD5%m=y zrnpjxWzPfqf4Bf8PCp;KurX3O6sySoNu#D#FR_%fVUMTi=>i|t0zKF4M$9K*;H0qG z7SJZh(lb3f?2?e#zXM9|vkqmW;U&V`O^;cb2eer5jJsKx_ZOW6UZU9>ZuABFcF8w{ zc=V476w|%n3gz>!&*=3Ax)(6F?R81P|8e_qM2y~u`yXo*)~oA|JEp&QN-)p){~g`Y zOXbI@>Wsbs9!CB}8QNZ=H?oAW29qy#_yzn4qPm2MT@(d!p8U77cpbyQ)o7fTFh!HX zmKS?ks{G%c@DtmEv@)sVoZ_W$b37Lbo@=<0NGDnJR-{dq<0^wTceskGiARKeu;Lap z1X_W;lX{*Tv!d6HI6qTdgIggT2!{bE81ewL)A5`v?JyOpzi7hUspYL`wimX!S$-3^$7LMO>PAb33ehJC;`#B5H z_7LHIcfsoM__1+<*Q)#bS7{X!>Gyk4*H2oek#3EuRtjroD}oz$zz9558V^Kd~+FYnwUHW|u&;$X@2iF?^A$ zZ@Q{vONZm}^UY9VWQT)H=PrEI01TObNL(c6E%3!M)4FUBF-FUUm0~k^z_mTX7D@{;; zJ0S15;!h=TQ(0CYAYsaBwX$|Cx|7;h#mi)BwBDD1u0-p9!mFyJX%ke+eXB$>mQgMQUoz?u9a$ z@KsqwEXH+I#{|9K7r_dl;H0`}9I!Bn0wE8sp}<9!C_W^+Q=#hg6e$?ak=E?NMYFL` z@FKznUQorfg^@oN7eyq5DB#L;YIZoGByg&h`^vwPVfLg{8GWJGo-cZF1~>J{xGp6h zIAY$_LKKanSJ~k$EkjR)n3a)vDQcX0pGU4Mh*9r2Qx*yPJBgtK$`g17EyXV7b@Kze zu=!q7Ig9Vgd$pBwMcgOsyP_5x(u-}Zz2@h1tdyOePEfy1{FY?jQ21zVrP0%P(lVHT zz8#9)zPLO1DN+$qTwN_4T5HP)We}&P?Y3ZcGz7Nm+YB~ll@TKCoJOeodVcCYR+JS< zambNA1~FZCuFAY!(dK;nWVN2Nh(p#v8n|#`Gz;Ct7e@Qyh}uXacGLmB8W^?0CclLO z+2r$0_8zojg7Rwid_?~AMS-$1Rp#`R#}}Z0N?7nWg$KLi%~W=N;C@QZRCt%!;j! z#>**QjT8MvMO{Kf=HuK`!;mXP85!?4J#7T)(=N|IFzo-+*jYx!(fnyUXo9=D1b26L zf($OfAz08PxVt;SZE%7FcXxMpcXu1!d6MV9yB~JXdFRZBIXzw7UENi8RsZBFsS%MB z9r=@Vo|D=OdA`{D5mLU70U8y#-0B!9#M0Jb7@1hq2aFqB?~~1Ylp(P^oiIPD_bhIj zWa5#?rlCn!a7|M8P8=g9B!>!pgw^v;&?T&i32OHA5i1nGxsg-jW&KB$9pY5|M9w1N z9cXsdm+?T+`XMFINUU)rS>j(UA$KpXfq$f15&pjjSA9H#nX>;h1n?g%YX80Yzrj~Q z|A(=X=MYBR6Y+H)xikku596QmA(xw!2+@6#)Pw#kq(smsL}=x)bJA=;{4-U`a_{} zG*}Q%fcFkoPm}rk)+E7CSkzxmy zS6artpiUqKB4A7LG;8G7FB_{WZog>50#2ccSB3$&2?273?lhMy?`DN{kVVeCHMWTA zq)AYB^i~*G6Ld|~yi6g3t(Q93WL}=sd)OIoJjPdNkNE!l(*I_sfoz(i#s_Usue zG;OU&e3i>o_unsc#z|wjXQe#iSu{~+qy39hN}Fp8oARkF_)^nc$VT^jzqz$(*y%zI zmO^Zo63)QR;xa@Z$1fCE=0(tnbWJ%M+1UY+xic|;9wD>cwhLva$*x!17+a zhq@rve5O?C;e|F-e>Zc(gJiTP{Vj}Z_*RgS7!Eg(fTB=uBWjP{1sY^AMoLQ4^nizB z=7qV+9MfuqGB!ikRd}3WyCPTF4{RtvaqrhxSzncR)#T3K6werfxAIJ%_@yRSj}j53 z|G=a4o#iH6RIIT7qVCS*lo8-nld&R&a~%P#FBEA^6B7~tvhf*seO&zV;4xB?JD>iy zmgeAEislxJ?iP-k-5kPh=Jm}i#I8Tp%LxfeW(Bb^q!_bUa>!^ox_y+7Qcvt>mlRD) zCd7j%f)g~?I93{udHewpL8_EE$x97aToR#~;?nHdemG6d7>XYv zZpjZ=ND}%u{krj;L{5`7B*N-_By51A_~dUunxH6FM%F6Xt;T5We|p@ODO;2@`o@ap z$H&V+w3(XTG6I{@?wxCotgmQI_X`4>u!b$X*cByLkJ zxD@h-cnCBbUsV9c6veUhKPO!FT{)s)KfKd1AsVzMpj}^5TqMRTfMd+!jyoUwC}AyC zctvA`F6@rQnIt^`sv}^|1D=ei;NRsW_pZIOiVU0jT(FRf#wXstYLrW*-(~awVF7Pd z>z9S_AxjXZTGkKpk4QL1f+?-wcO#t*G@MWBo9kU)(xXY)E>+5$`(FJuawF92s|W zc=iY?cPx5(`kDZb!C4LoFBch#KdCH0WM_@jdu!=u;H>;GA2_222nQ0|;-~!`CBgoC zQWwxy>Xg!6;kWVua11ZH$b8J`@wlv`Xm)S^svX zftkzoyf$A>w{KoVgUQ+55E zrnQaged*+QHB!wzllsJlE0SMY`mQZe8h?hWkjvu+7&z5}Phf|<)A?HDHcx;U+1^-& zytp`2NeOy|%Hz!PRsgL0=bJaW+;j4oeMcr_F292AKPJb+o>1=Q1>oUVoBZruw;vioCxgXg+BtB!PkuNuz$MWW7 z8CTp2GuF)5J=8XtaX@!XO`t|n{~zM!Hd2ry^Fds<$-V+-ZLVVY!QT5YN96lX{dyr?rc9n5b|8L+oKxA~|`J;c! z=#Tb zUOuO=39Ii6ymAj0;2|OBsV}77+!tU`J`iU4gN~LGY`s%`m*0PAne-@EQYK#AhuBRx zU+?O0=C_w0Z&h;s#t!w?h`@hM9@UG-8`N1N5oy&cr}OF zzIJE~el!^)0?}QO$G8vb1pf1-q^}8{$r1jD`gMUJ4f4X8)d8@Ub$OMhh;py7Ve#^W16JF$WA1C-}z=MgSv&qET3-HxMslz+~ty6*PGZWSh5;1`9WhS4Hi{-HAxTCCgwgPZi& zBKhVrA-TUI@w5|PnROr@96DPR2+s~YNE)N`TfWcad!NuH$8cWm2hOWR$1*^TAr3Nn3BW6xKB@k5}W#HSBw zuL!+@67LZfN>lEF=7`A02!%mz6gInD*ACuW#idfIJztrSiWagy&uC03kfFtA zC!^lbvXwzF5R|@~qQa!l!Z@}I#Yu%uYO)1VH0GU%T`9%WoD?Fzzs_v9d2s-B&%`lO z%RYmIw{s%}XrO;9A;Gz0Qi%PQI=e-9wB}4c!hmB}w`!8wr^mA_s%K!CsQ zVdoc$zu|WZov?zT*Qx2})-_xo@47EHGi(()5}9q79mJl;qz$JNSHiE<%TbkAzT`J+ ziNVriIL}ZboJx6LX_HWw_yO1OcZI~Q`7w=`EkPhPJhT@xlsI;+L0YIH(WSQq#dB2R z4pX|^wmClS%e|tbbK!*^?n}1hD9V2POAkYkaBX5H3Eza8!mYFU_WHcoo;eh>#ld-M z=pruPWaJ;YxvVOLzIu-SJ2HhpMJ)eE|3XauM}X1)HeCK+62#*GNmB6x*V;XHqcO?& zFLtzlvH=;r$}){)J#BZsIX~ES1kCYmuZz*;mpKEYM(UP=2a~}Nv{u5&roJ|%$1@wm z*A|pF3kEN49*I5vGR*q>n8si4*YBdc6#wX`zH6UD%tqxA9K!hJ!lJi`Im-tcOhJ{O zrD)D-Fc4#BUPG^vrRwgLWwE_Y;0gID(?$)>#TBt3;A#7vlL1;`nGZL_Z1v4hrXnM@dvNZ5*k?nD#9#ZUGk*JjZ}>g55!-CP*n zm+3u#rvlPl>*$Uxft35WfK@qs?z^!%TNPnTQATNH=HySKiM0z#-{T%1NqM;Rt722_ z6a#;hA+D!N&0@~Xe=2y=yNwCKXqD;yI`x4AL-OBIWUCjqU$o1}f^I(p&7{)xif9Qg z-!i~}-E-%5>u9ov<57322lULaH7ykk`CFL|*dx>ypp_R5LUG>C#2}j|%DddBNMJ`>Bf~q7%J* zTlI4AW6yrD)YaUq%40MIb2mbiXZ&B0FI61M{63&|5xGB3txreN8+H|4a$>rT%FGct7amS!nNpvJDXoex3Q#%B{ZzaaoKZed?TyLf>CIYSjAMkHxg*= zm{46H zLF#*%5(RWPTm}(k-mk(zmI@0>t|#d@g!SLW=tpFqY~-)*3nTl8{k&-}`*Hv{*3vxI zZVOJTf_r&)aT&}g@ce+~{rjD;$+qq|Q>ACmJ|u+bVCOKgk$d#!=IwH@{=#z+@52J} zct1(aWvj{KBPJ>)QA_$g`O(S5P*_5Xn)7r-RDf@@t zXC7f~%=JWx=RO@y_V0(GGlRJeBWDJ0FWiP*%--8Id}0|}Lm-Pv0>2KNtTrdz-QD?v zkPs0!TywKw_&QZ6#jSJCmn{zI6`F-CGVIn5HQAP}c_bxQQ8 zuhnHd3yXuuQy)ue)*+%7!BWLwRdn&nmu-=`;$!nacakNo>)t8Gle}HH*ZS1>p=peQ z4`(QvBOc`X7JhJIF4O8vzl+ndir{cjNu(cpLS3hf>)}u@5~cRoDVS$>?2VS;5j;iW z%=)l~ly@demwNIW6qbw5!U4fq(Kn>!%IW?)CqLBfM5ou=c>`|& z_@!EFoYB$Io7>ydH*OXd5gJTjQ&XyO-}W=g&f}wPk~$Hx_t{;8X{xB!b;CYL;H|Q* zr_Qg-vKF2`i{>|ZVHGK%{qj0IVvky;R>pp_E|G>a&&ui{TcKznD^P55&zpn}Ez?59 zjyRe^Q1(&ij=l9E5^Ew|%s?pIzAZ8*t5|JSS$q>=G&zee{(&QYqLGuvpKFQVT=nq? ze&|?7#?umI?#O89*ONUtfc#r-FilT^{W4co`#?^Xp1g(VcgnC)L>8CZFOKbJJ&p>% z!X6a8evh!3c46_mJtqcg35zz_$8x5jDBnd?<%WEWLZPs!_=1Mk@RNO~SZ|t0H!B#a zK(7J4C8ZkHmq(i&&AiDyN8KYdqeP66UWgMEXPOb zr|PsyE=vhe4lxDamCgRCLM&)V)zt2|@0SvKCVJQ6?BGRkU0nM&=1y&e0J@IVxv>q- z<$?9dQ0}tSi#!Se*HiS~_FWbO-Dq1Et@|IHhtmM(bEAiA{(-lq_$oANHfF!;9({Oz z_dzc=>PJrA2u)3llY_#B=HQcB{z2hXusz@Rn;pN#T1R>wz^f?GBU;o&?jWQj{<#yD zO_(J6$XSjC|9!45N}u>K<59{ zhCd&hLMDDSraK*99DsGw@2pA~g>J>tcd||}5@W#cqd_9gsh&PswzG_tAjo^N=5@7h z(@lpTu-AX-bQ{xjhUZxCq=K)(VjeW;?GDulHsMzg0BdNYD``SAc^=Z~1*;I2qfIo9 zT|8+WkAqmL0xHk#zB~v!sko4bH{}siIN1{+pxHCsT6pN^FiXViKQx ze9pj5Cyi*Fptw)epV>_;@(Medhn>{dQ`ouD;b;%Q3B%CFm88uZ^r*$y^~3+p;3&N6 z%U?;%?qI*TEA7mB)0lzjH7&~~z-QQ%V^TM&i^)}bV8ca&x z9lO&kQ|0tJ_<@9i*4tB7h%z3+sZmP=AHMih*QU+RszmJIW{U>*#gicUU@S!cbq~ucuU*cl5d9EdXC49pk5^KvE zI!+7y;Acs)0+>==U1LqL{UghZg(mV}h188MwjtxfmY4G$KKSgs&7#zfM(HXg9RcB6 z>6*o8i_WH^A_W^J!;yYTQqv)Np0KRyzTs3paL}^>1>d)BgIUIkJJ!>J9uwc&SY-+& zmK;M)tzQ>4Sn^37*tbIZA0%D_Fq&yk*0?46sl0XhJI$tx-F{%X)jAv~tdggk*P^x< zas7}deGM!##m)?Cqz{iTeTTWCfX7o0L6J!IJdDV>t_!aVMi(Z8Dk58B35LFVhYFCD z5LF|okvBiq4?4%qyK0(d0lS2L2R+yLvqN(l=tx|sA@oYaD&v?9md_QxOr}6vJH>zi z?6X;!qP<8610F?q&ffqEi6OJhIkWXROMYGJvGN=^20R%gDVh;*0>s~6e9ZKQ5s}}|JGsra<=*~T&V1z zkN&ZAlheJSn>dl1L27kc?y|x&`Miw6?U68oU2lbYdC;ppBb9k9zbae#tE|f9CY(jE zpdG_i`;q9|gg{*3sWo$W$7t@@=TED;2#7D5^s#pHF|QKTvFGWsY6Qj?7)pSb7sOHp z%Py<1l^~wYV%nAIFGOm7C67GK)jYD{GpDF+MLztE*zx4MAUY~gQPfR6eP*IqoU4*d+H z9X$qB{fym>YLbv3Km_sCaUpR24#4%R$dc!K?dV4gSuavvM=wv;_r2U1{!4-)UV7hm zKD$})=YJu1p43QBnWfavRelg92sFc@G$*XOVTBy0c*{lFu_fuM{N&~}9rkz$qh^3h z?X=4vCWt&|uspcBs=f{1T`Z8n3SkPk2eAcB}{J2O6?|j;E7OLD*PV{;NUKcU`wcaM#nVIUv zdA;aqMXID7Zv&jU?|SSZ76gmw%aAFQ_Vfn8dokwUp8fDzmPZ^GYyWM*pO1CRWmG$z zR8e_T`JtQiGCD)5C81y;so#0a!uuXf<3lpm9O)D2?{+PXJQ>YU^Tu5%74KdCX-M~) z_g09veSBT~a7DW#RR5m)`t$ZLs?caAiN{IXlgwCFEE5L|b#9-xK}f9nkvi7>y1K?=lK!l&bmvW4^7OE1aa)L*F9*L73b9hs`!@ifo z#RlO83D-W#5~WZv9R_qUB{pJ9vF&LfHgA49=>{C5qP;UQR|j zvgEzTTABotP1j>LX`=yBwqix$5mE3#@eW}NUHpO{jg@D}fTO<{=t31OVmC5boor@) zgzx)%xC{5HqD#J;!C0s>C;WhMi=JeR?o}hy?@iExt}|u|0Y``{s`!)7?>hj>@qA$p z8G<0KdR7)HGI=yrKTI=fK>SBs(f%gh=U87)!LWbD22MEtHzj4uZk9++1yb&u$ z{A!NaYFrV(i*5osa$E5{E4i=gda{K_iQb^pO%Q$~WMxc^&H#F-bq@yi>|RGU-cAaY zM!?m7rG|KzHe8zkofOc^zGJ|iY!{aGR1+Vflp1%p190|>Eco$QlBx{PMn=_Ezqi)K z=e0&_$0q${hTZOnt~~-wBT5w)ig8L4mV># zr$T$ggt>f}h6e3uW!vHxmZ0h51f>I>#dq4g!W1T}3sL#e=cJQ2MFKh$bb$}?*nOFV zLDy?P(Kz&|ub-doom%BAfoZkvwQBDJ`o~pV4dKg@B^2droStG?qjP#=Q?h4%enX?Up$4TDFT0H8 zs{=B69ko$Z3rncciFJ+p#ynwt2~2^TOP>&Ikh7BNV+Afh30jt}m$Yi0?Wrgx=M9YP z46Sb2$57+Y>gC=8MI(cZf<>*?Zrc+kLzFxgc3KTe*GmSL{9swN4cVjHs8O1a?*@OzklUNHLJ zRzgY{8m~+QzdaDnN{{Eoo=n)ix`nyT)!{a)eH8-3)oI4$FQcuk%wp*hECpNJeD zG>t3Mt*`pFugGP$7Gdd>N|qQgE%cEC# z;*7T?9|$0?KWmYYl+$=!V%3TuceJVv6VBzuk+eSiCcz#C27 zIm@hL$(~62Lz#%WIv@9W)WcQm(ZYQ`yRY@%djh0Si=;VJftR0rWULJX0#t>{a?S9W zvzqfh+f1gxREu#el9mMZ80}Zr2FN~)5RRr`Zz`F8Ft=f$tRf`UMRQ~od5TMBida?m z+a?C&>v6fh7yS799Jf!9=hNO@EpA=U0-wm8s(+poUk$l%Igv)z@F9_ zeS@ULWR*?ks*j=4HMDWUHmb)yW64{C#pyPi**TzZjLm%>VWw|=w;7kLGf$}ow^S$S zfw(hW{OHUm0_Y9mg_G5~3dLp$$;YDX8^AIZZp@r<3ca6c9CMeN!hVW%Ey-mBURtJ( zYae*RE>*%Rr1Z6kp(|P*(BpJ>%%Nd0;;T}H3UW=RV7s+lJHsiu{Vof>A!@Z4gCWou z#qM;mrv0fgoR0%rCN8w6&}vc93RmDecS`!CLMbd5@%(5sHUCBPeBpy^!-ef`K3YFg zS2g2Nbad^v-H%oHq{nELI7~zkZqKYO+!k9~rKuHo;AP%A+B_)+kM&qILYd_m6yBZS zbqMlAxR0ZTW;(HCc=;bT>jQ;6E!`1;ywL-Lv7iyD8@22Gb^Fy_RAE7J&nn5AC(1Gm zIElg$nRzKCpC8Lp?#EswC??=^1th#VIBMMib^dcQ&?etr1xj2C@B5^ys3Ux1)@-i^ zv4<~-d*3)Ya+rI&JmNP#MLw}tC)HF?-wAZ}QXV!3PhJo2;$zg4n>I?iSRmsSq8&O3 z?n}#GPABxpnC|*vWjq#8&`OKHIV08Iu%o$szj2(KhTKqnwDDe?=}_Y4oM6{ zvu?6KCnRs3@AP)c%=Yx3MCeZetpHLHlI**~fZuNrmg9eqa43A9?`4TVe)Q+F#y~Uh zD(H;=#>$PN^YxgW$FpkiRXSCG7r(OFU_UXQG0Y!MsA-TKfaGGOv3=wf4}sHmDdvA3 zA^la`dMbW`G=VTJ_A6nH%QIb>Bku#lntQ_DCH9p?s6}LWbteQr1Uq<54(bo6Hi`53Nd`HJFG$VFH%Vdk_0w2pKh&9W}?+t>r5vx(DA?! zh>tl(;^d*<6j1uQIKsWT6(>ewp0|xgTzsd@718BnAL3!ixkFoN1Jxrg&jcgynX7wR zneGfB+`oz3GAjYzIyoSa@0?_BQKD2{&k#XikZUh}Wjww`+{T1WpK8z&G#OzeVkc5u|0SVzxydBu74Z} z*Ek5Tj_SUuZdN|bV;sz*Y}}M{@!J~SSU*H(9^%O8jQAxQ`%>_?c=!a|ZEKun6 zY?~Jg3m}o4bsx%`oXVK&Fao=`XNWV%&J2bs4T|MDEgO;Z;T$wQV^D5lGY&N8 zySd?%J&;vtem^l}77?qDzKnni;w|y!mwR2`4|QkLIzB;O#KI@HLnFd3ygSw20x{hA zDAXwTFEs4=c_PYwua~%`HwidymT}6jEa$uc?chdgrVXf?y+}g2+rjz$m_cz(er{$VgaFFzuE~z9 z<96s*Kh4;FzK{KaZoaDGVO1x2GsSaz-Ws|f>{CtaKWj&xhWEwvAcJAW!|k0=`ct$rb`#E#)DLv%t#<8jwc>r&dQBS+v#%RBj zaFK!!ZnTf0KXr40Gde9x8DYyvK25tgre?=cyycTI0CK+XTSMzhe0TAkkTbN4q*;HZ zD@%FzC}&x$*i2XO6x?r-c@qm}IUP~mogsF~0A~w|Ub~?gyUBRTi&mWp?Ha)-G@kjg zkGHXuYX(I9?#5w#mGAcFRUS|>@~&oCQ-eKD_=9C^zE5yu*Mta@+23KU8Eq8M47um@ zl0>|b71Ffd=m_(Q9ui!Gv5pk(-#~F(9w4Q`Af?w>lNm+%m2q2&#W9j3yPw)Gd)6Fu zF!0`WFvsXsISD917GHl3L79hg?GCPTg`_kKK;S5hR6lGuc>K!h5+oRN?!l4-B)q9w z#bncvpjPf Tyui452mZ)PDoT`#8T$P{6=#Uk literal 0 HcmV?d00001 diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst new file mode 100644 index 000000000..dd1dd3a62 --- /dev/null +++ b/docs/topics/developer-tools.rst @@ -0,0 +1,248 @@ +.. _topics-developer-tools: + +========================== +Using your browser's Developer Tools for scraping +========================== + +Here is a general guide on how to use your browser's Developer Tools +to ease the scraping process. Today almost all browsers come with +built in `Developer Tools`_ and although we will use Firefox in this +guide, the concepts are applicable to any other browser. + +In this guide we'll introduce the basic tools to use from a browser's +Developer Tools by scraping `quotes.toscrape.com`_. + +.. _topics-livedom: + +Caveats with inspecting the live browser DOM +============================================ + +Since Developer Tools operate on a live browser DOM, what you'll actually see +when inspecting the page source is not the original HTML, but a modified one +after applying some browser clean up and executing Javascript code. Firefox, +in particular, is known for adding ```` elements to tables. Scrapy, on +the other hand, does not modify the original page HTML, so you won't be able to +extract any data if you use ```` in your XPath expressions. + +Therefore, you should keep in mind the following things: + +* Disable Javascript while inspecting the DOM looking for XPaths to be + used in Scrapy + +* Never use full XPath paths, use relative and clever ones based on attributes + (such as ``id``, ``class``, ``width``, etc) or any identifying features like + ``contains(@href, 'image')``. + +* Never include ```` elements in your XPath expressions unless you + really know what you're doing + +.. _topics-inspector: + +Inspecting a website +=================================== + +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's take a +look at the `quotes.toscrape.com`_-site. + +On the site we have a total of ten quotes from various authors with specific +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. + +Instead of viewing the whole source code for the page, we can simply right click +on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. +In it you should see something like this: + +.. image:: _images/inspector_01.png + :width: 777 + :height: 469 + :alt: Firefox's Inspector-tool + +The interesting part for us is this: + +.. code-block:: html + +
+ (...) + (...) +
(...)
+
+ +If you hover over the first ``div`` directly above the ``span``-tag highlighted +in the screenshot, you'll see that the corresponding section of the webpage gets +highlighted as well. So now we have a section, but we can't find our quote text +anywhere. + +The advantage of the `Inspector` is that it automatically expands and collapses +sections and tags of a webpage, which greatly improves readability. You can +expand and collapse a tag by clicking on the arrow in front of it or by double +clicking directly on the tag. If we expand the ``span``-tag with the ``class= +"text"`` we will see the quote-text we clicked on. The `Inspector` lets you +copy XPaths to selected elements. Let's try it out: Right-click on the ``span``- +tag, select ``Copy > XPath`` and paste it in the scrapy shell like so:: + + >>> scrapy shell "http://quotes.toscrape.com/" + (...) + >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').extract() + ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”] + +Adding ``text()`` at the end we are able to extract the first quote with this +basic selector. But this XPath is not really that clever. All it does is +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: + +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div``-tag we have eight identical ``div``-tags, each with the +same attributes as our first. If we expand any of them, we'll see the same +structure as with our first quote: Two ``span``-tags and one ``div``-tag. We can +expand each ``span``-tag with the ``class="text"`` inside our ``div``-tags and +see each quote. With this knowledge we can refine our XPath: Instead of a path +to follow, we'll simply select all ``span``-tags with the ``class="text"``:: + + >>> response.xpath('//span[@class="text"]/text()').extract() + ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”, + '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', + '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', + (...)] + +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily +complex and by simply constructing an XPath with the ``class="text"`` we +were able to extract all quotes in one line. + +The `Inspector` has a lot of other helpful features, such as searching in the +source code or directly scrolling to an element you selected. Let's demonstrate +a use case: + +Say you want to find the ``Next``-button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li``-tag with the ``class="text"``, the second the text +of an ``a``-tag. Right click on the ``a``-tag and select ``Scroll into View``. +If you hover over the tag, you'll see the button highlighted. From here +we could easily create a :ref:`Link Extract ` to +follow the pagination. On a simple site such as this, there may not be +the need to find an element visually but the ``Scroll into View`` function +can be quite useful on complex sites. + +.. _topics-network-tool: + +The Network-tool +================ +While scraping you may come across dynamic webpages where some parts +of the page is loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools +greatly facilitates this task. To demonstrate the Network-tool, let's +take a look at the page `quotes.toscrape.com/scroll`_. + +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next``-button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead +we'll check another quite useful command from the scrapy shell:: + + >>> scrapy shell "quotes.toscrape.com/scroll" + (...) + >>> view(response) + +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. + +.. image:: _images/network_01.png + :width: 777 + :height: 296 + :alt: Response from quotes.toscrape.com/scroll + +The ``view(response)``-command let's us view the response our +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, +the login-button and the footer, but the quotes are missing. This +tells us that the quotes are being loaded from a different request +than ``quotes.toscrape/scroll``. + +If you click on the ``Network``-tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the +log is automatically cleared each time you navigate to a different +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. + +If we reload the page now, you'll see the log get populated with six +new requests. + +.. image:: _images/network_02.png + :width: 777 + :height: 241 + :alt: Network tab with persistent logs and requests + +Here we see every request that has been made when reloading the page +and can inspect each request and its response. So let's find out +where our quotes are coming from: + +First click on the request with the name ``scroll``. On the right +you can now inspect the request. In ``Headers`` you'll find details +about the request headers, such as the URL, the method, the IP-address, +and so on. We'll ignore the other tabs and click directly on ``Reponse``. + +What you should see in the ``Preview``-pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. + +If we click on this request, we see that the request URL is +``http://quotes.toscrape.com/api/quotes?page=1`` and the response +is a JSON-object that contains our quotes. We can also right-click +on the request and open ``Open in new tab`` to get a better overview. + +.. image:: _images/network_03.png + :width: 777 + :height: 375 + :alt: JSON-object returned from the quotes.toscrape API + +With this response we can now easily parse the JSON-object and +also request each page to get every quote on the site:: + + import scrapy + import json + + + class QuoteSpider(scrapy.Spider): + name = 'quote' + allowed_domains = ['quotes.toscrape.com'] + page = 1 + start_urls = ['http://quotes.toscrape.com/api/quotes?page=1] + + def parse(self, response): + data = json.loads(response.text) + for quote in data["quotes"]: + quote = quote["text"] + print(quote) + if data["has_next"]: + self.page += 1 + url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + yield scrapy.Request(url=url, callback=self.parse) + +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. +We iterate through the ``quotes`` and print out the ``quote["text"]``. +If the handy ``has_next``-element is ``true`` (try loading +`http://quotes.toscrape.com/api/quotes?page=10`_ in your browser or a +page-number greater than 10), we increment the ``page``-attribute +and ``yield`` a new request, inserting the incremented page-number +into our ``url``. + +You can see that with a few inspections in the `Network`-tool we +were able to easily replicate the dynamic requests of the scrolling +functionality of the page. Crawling dynamic pages can be quite +daunting and pages can be very complex, but it (mostly) boils down +to identifying the correct request and replicating it in your spider. + +.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools +.. _quotes.toscrape.com: http://quotes.toscrape.com +.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/ + diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst deleted file mode 100644 index 4ea8d3bd0..000000000 --- a/docs/topics/firebug.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. _topics-firebug: - -========================== -Using Firebug for scraping -========================== - -.. note:: Google Directory, the example website used in this guide is no longer - available as it `has been shut down by Google`_. The concepts in this guide - are still valid though. If you want to update this guide to use a new - (working) site, your contribution will be more than welcome!. See :ref:`topics-contributing` - for information on how to do so. - -Introduction -============ - -This document explains how to use `Firebug`_ (a Firefox add-on) to make the -scraping process easier and more fun. For other useful Firefox add-ons see -:ref:`topics-firefox-addons`. There are some caveats with using Firefox add-ons -to inspect pages, see :ref:`topics-firefox-livedom`. - -In this example, we'll show how to use `Firebug`_ to scrape data from the -`Google Directory`_, which contains the same data as the `Open Directory -Project`_ used in the :ref:`tutorial ` but with a different -face. - -.. _Firebug: https://getfirebug.com/ -.. _Google Directory: http://directory.google.com/ -.. _Open Directory Project: http://www.dmoz.org - -Firebug comes with a very useful feature called `Inspect Element`_ which allows -you to inspect the HTML code of the different page elements just by hovering -your mouse over them. Otherwise you would have to search for the tags manually -through the HTML body which can be a very tedious task. - -.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA - -In the following screenshot you can see the `Inspect Element`_ tool in action. - -.. image:: _images/firebug1.png - :width: 913 - :height: 600 - :alt: Inspecting elements with Firebug - -At first sight, we can see that the directory is divided in categories, which -are also divided in subcategories. - -However, it seems that there are more subcategories than the ones being shown -in this page, so we'll keep looking: - -.. image:: _images/firebug2.png - :width: 819 - :height: 629 - :alt: Inspecting elements with Firebug - -As expected, the subcategories contain links to other subcategories, and also -links to actual websites, which is the purpose of the directory. - -Getting links to follow -======================= - -By looking at the category URLs we can see they share a pattern: - - http://directory.google.com/Category/Subcategory/Another_Subcategory - -Once we know that, we are able to construct a regular expression to follow -those links. For example, the following one:: - - directory\.google\.com/[A-Z][a-zA-Z_/]+$ - -So, based on that regular expression we can create the first crawling rule:: - - Rule(LinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$', ), - 'parse_category', - follow=True, - ), - -The :class:`~scrapy.spiders.Rule` object instructs -:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the -category links. ``parse_category`` will be a method of the spider which will -process and extract data from those pages. - -This is how the spider would look so far:: - - from scrapy.linkextractors import LinkExtractor - from scrapy.spiders import CrawlSpider, Rule - - class GoogleDirectorySpider(CrawlSpider): - name = 'directory.google.com' - allowed_domains = ['directory.google.com'] - start_urls = ['http://directory.google.com/'] - - rules = ( - Rule(LinkExtractor(allow='directory\.google\.com/[A-Z][a-zA-Z_/]+$'), - 'parse_category', follow=True, - ), - ) - - def parse_category(self, response): - # write the category page data extraction code here - pass - - -Extracting the data -=================== - -Now we're going to write the code to extract data from those pages. - -With the help of Firebug, we'll take a look at some page containing links to -websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can -extract those links using :ref:`Selectors `. We'll also -use the :ref:`Scrapy shell ` to test those XPath's and make sure -they work as we expect. - -.. image:: _images/firebug3.png - :width: 965 - :height: 751 - :alt: Inspecting elements with Firebug - -As you can see, the page markup is not very descriptive: the elements don't -contain ``id``, ``class`` or any attribute that clearly identifies them, so -we'll use the ranking bars as a reference point to select the data to extract -when we construct our XPaths. - -After using FireBug, we can see that each link is inside a ``td`` tag, which is -itself inside a ``tr`` tag that also contains the link's ranking bar (in -another ``td``). - -So we can select the ranking bar, then find its parent (the ``tr``), and then -finally, the link's ``td`` (which contains the data we want to scrape). - -This results in the following XPath:: - - //td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td//a - -It's important to use the :ref:`Scrapy shell ` to test these -complex XPath expressions and make sure they work as expected. - -Basically, that expression will look for the ranking bar's ``td`` element, and -then select any ``td`` element who has a descendant ``a`` element whose -``href`` attribute contains the string ``#pagerank``" - -Of course, this is not the only XPath, and maybe not the simpler one to select -that data. Another approach could be, for example, to find any ``font`` tags -that have that grey colour of the links, - -Finally, we can write our ``parse_category()`` method:: - - def parse_category(self, response): - # The path to website links in directory page - links = response.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') - - for link in links: - item = DirectoryItem() - item['name'] = link.xpath('a/text()').extract() - item['url'] = link.xpath('a/@href').extract() - item['description'] = link.xpath('font[2]/text()').extract() - yield item - - -Be aware that you may find some elements which appear in Firebug but -not in the original HTML, such as the typical case of ```` -elements. - -or tags which Therefer in page HTML -sources may on Firebug inspects the live DOM - -.. _has been shut down by Google: https://searchenginewatch.com/sew/news/2096661/google-directory-shut diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst deleted file mode 100644 index 2c85848be..000000000 --- a/docs/topics/firefox.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. _topics-firefox: - -========================== -Using Firefox for scraping -========================== - -Here is a list of tips and advice on using Firefox for scraping, along with a -list of useful Firefox add-ons to ease the scraping process. - -.. _topics-firefox-livedom: - -Caveats with inspecting the live browser DOM -============================================ - -Since Firefox add-ons operate on a live browser DOM, what you'll actually see -when inspecting the page source is not the original HTML, but a modified one -after applying some browser clean up and executing Javascript code. Firefox, -in particular, is known for adding ```` elements to tables. Scrapy, on -the other hand, does not modify the original page HTML, so you won't be able to -extract any data if you use ```` in your XPath expressions. - -Therefore, you should keep in mind the following things when working with -Firefox and XPath: - -* Disable Firefox Javascript while inspecting the DOM looking for XPaths to be - used in Scrapy - -* Never use full XPath paths, use relative and clever ones based on attributes - (such as ``id``, ``class``, ``width``, etc) or any identifying features like - ``contains(@href, 'image')``. - -* Never include ```` elements in your XPath expressions unless you - really know what you're doing - -.. _topics-firefox-addons: - -Useful Firefox add-ons for scraping -=================================== - -Firebug -------- - -`Firebug`_ is a widely known tool among web developers and it's also very -useful for scraping. In particular, its `Inspect Element`_ feature comes very -handy when you need to construct the XPaths for extracting data because it -allows you to view the HTML code of each page element while moving your mouse -over it. - -See :ref:`topics-firebug` for a detailed guide on how to use Firebug with -Scrapy. - -XPather -------- - -`XPather`_ allows you to test XPath expressions directly on the pages. - -XPath Checker -------------- - -`XPath Checker`_ is another Firefox add-on for testing XPaths on your pages. - -Tamper Data ------------ - -`Tamper Data`_ is a Firefox add-on which allows you to view and modify the HTTP -request headers sent by Firefox. Firebug also allows to view HTTP headers, but -not to modify them. - -Firecookie ----------- - -`Firecookie`_ makes it easier to view and manage cookies. You can use this -extension to create a new cookie, delete existing cookies, see a list of cookies -for the current site, manage cookies permissions and a lot more. - -.. _Firebug: https://getfirebug.com/ -.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA -.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ -.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/ -.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/ -.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/ - From 3a71e7dbce94e6056a5258c59df0150a94ddc187 Mon Sep 17 00:00:00 2001 From: testingcan Date: Wed, 22 Aug 2018 16:57:51 +0200 Subject: [PATCH 0157/1853] Increased length of "=" --- docs/topics/developer-tools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index dd1dd3a62..aa4050948 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -1,8 +1,8 @@ .. _topics-developer-tools: -========================== +================================================= Using your browser's Developer Tools for scraping -========================== +================================================= Here is a general guide on how to use your browser's Developer Tools to ease the scraping process. Today almost all browsers come with From 4d3aaabbcaa3a39ba56f443ccfe8a60888223013 Mon Sep 17 00:00:00 2001 From: Raphael Wuillemier Date: Thu, 23 Aug 2018 12:40:31 +0200 Subject: [PATCH 0158/1853] Updated code, added code snippets and improved readability --- docs/topics/developer-tools.rst | 84 ++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index aa4050948..6bd2c052a 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -27,7 +27,7 @@ extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things: * Disable Javascript while inspecting the DOM looking for XPaths to be - used in Scrapy + used in Scrapy (in the Developer Tools settings click `Disable JavaScript`) * Never use full XPath paths, use relative and clever ones based on attributes (such as ``id``, ``class``, ``width``, etc) or any identifying features like @@ -43,8 +43,8 @@ Inspecting a website By far the most handy feature of the Developer Tools is the `Inspector` feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's take a -look at the `quotes.toscrape.com`_-site. +any webpage. To demonstrate the Inspector, let's look at the +`quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes @@ -69,7 +69,7 @@ The interesting part for us is this:
(...)
-If you hover over the first ``div`` directly above the ``span``-tag highlighted +If you hover over the first ``div`` directly above the ``span`` tag highlighted in the screenshot, you'll see that the corresponding section of the webpage gets highlighted as well. So now we have a section, but we can't find our quote text anywhere. @@ -77,14 +77,14 @@ anywhere. The advantage of the `Inspector` is that it automatically expands and collapses sections and tags of a webpage, which greatly improves readability. You can expand and collapse a tag by clicking on the arrow in front of it or by double -clicking directly on the tag. If we expand the ``span``-tag with the ``class= +clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you -copy XPaths to selected elements. Let's try it out: Right-click on the ``span``- +copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` tag, select ``Copy > XPath`` and paste it in the scrapy shell like so:: - >>> scrapy shell "http://quotes.toscrape.com/" + $ scrapy shell "http://quotes.toscrape.com/" (...) - >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').extract() + >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”] Adding ``text()`` at the end we are able to extract the first quote with this @@ -93,14 +93,28 @@ go down a desired path in the source code starting from ``html``. So let's see if we can refine our XPath a bit: If we check the `Inspector` again we'll see that directly beneath our -expanded ``div``-tag we have eight identical ``div``-tags, each with the +expanded ``div`` tag we have nine identical ``div`` tags, each with the same attributes as our first. If we expand any of them, we'll see the same -structure as with our first quote: Two ``span``-tags and one ``div``-tag. We can -expand each ``span``-tag with the ``class="text"`` inside our ``div``-tags and -see each quote. With this knowledge we can refine our XPath: Instead of a path -to follow, we'll simply select all ``span``-tags with the ``class="text"``:: +structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +see each quote: - >>> response.xpath('//span[@class="text"]/text()').extract() +.. code-block:: html + +
+ + “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.” + + (...) +
(...)
+
+ + +With this knowledge we can refine our XPath: Instead of a path to follow, +we'll simply select all ``span`` tags with the ``class="text"`` by using +the `has-class-extension`_:: + + >>> response.xpath('//span[has-class("text")]/text()').getall() ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”, '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', @@ -109,40 +123,45 @@ to follow, we'll simply select all ``span``-tags with the ``class="text"``:: And with one simple, cleverer XPath we are able to extract all quotes from the page. We could have constructed a loop over our first XPath to increase the number of the last ``div``, but this would have been unnecessarily -complex and by simply constructing an XPath with the ``class="text"`` we -were able to extract all quotes in one line. +complex and by simply constructing an XPath with ``has-class("text")`` +we were able to extract all quotes in one line. The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate a use case: -Say you want to find the ``Next``-button on the page. Type ``Next`` into the +Say you want to find the ``Next`` button on the page. Type ``Next`` into the search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li``-tag with the ``class="text"``, the second the text -of an ``a``-tag. Right click on the ``a``-tag and select ``Scroll into View``. +The first is a ``li`` tag with the ``class="text"``, the second the text +of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extract ` to +we could easily create a :ref:`Link Extractor ` to follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function can be quite useful on complex sites. +Note that the search bar can also be used to search for and test CSS +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. + .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page is loaded dynamically through multiple requests. While +of the page are loaded dynamically through multiple requests. While this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's take a look at the page `quotes.toscrape.com/scroll`_. The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next``-button, the page +but instead of the above-mentioned ``Next`` button, the page automatically loads new quotes when you scroll to the bottom. We could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the scrapy shell:: - >>> scrapy shell "quotes.toscrape.com/scroll" + $ scrapy shell "quotes.toscrape.com/scroll" (...) >>> view(response) @@ -155,14 +174,14 @@ bar with the word ``Loading...``. :height: 296 :alt: Response from quotes.toscrape.com/scroll -The ``view(response)``-command let's us view the response our +The ``view(response)`` command let's us view the response our shell or later our spider receives from the server. Here we see that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request than ``quotes.toscrape/scroll``. -If you click on the ``Network``-tab, you will probably only see +If you click on the ``Network`` tab, you will probably only see two entries. The first thing we do is enable persistent logs by clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different @@ -186,8 +205,8 @@ you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Reponse``. -What you should see in the ``Preview``-pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response`` in the +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the shell. Accordingly the ``type`` of the request in the log is ``html``. The other requests have types like ``css`` or ``js``, but what interests us is the one request called ``quotes?page=1`` with the @@ -219,8 +238,7 @@ also request each page to get every quote on the site:: def parse(self, response): data = json.loads(response.text) for quote in data["quotes"]: - quote = quote["text"] - print(quote) + yield {"quote": quote["text"] if data["has_next"]: self.page += 1 url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) @@ -230,9 +248,9 @@ This spider starts at the first page of the quotes-API. With each response, we parse the ``response.text`` and assign it to ``data``. This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next``-element is ``true`` (try loading -`http://quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page``-attribute +If the handy ``has_next`` element is ``true`` (try loading +`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a +page-number greater than 10), we increment the ``page`` attribute and ``yield`` a new request, inserting the incremented page-number into our ``url``. @@ -245,4 +263,6 @@ to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools .. _quotes.toscrape.com: http://quotes.toscrape.com .. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/ +.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 +.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions From e98e7f8506b401f80fa193faa741c2094d7b62d7 Mon Sep 17 00:00:00 2001 From: testingcan Date: Thu, 23 Aug 2018 14:50:49 +0200 Subject: [PATCH 0159/1853] Added missing curly brace --- docs/topics/developer-tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 6bd2c052a..c1976258d 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -238,7 +238,7 @@ also request each page to get every quote on the site:: def parse(self, response): data = json.loads(response.text) for quote in data["quotes"]: - yield {"quote": quote["text"] + yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) From 79de3d569a3b50b3a07ced99d8e482862a979e88 Mon Sep 17 00:00:00 2001 From: Raphael Wuillemier Date: Thu, 23 Aug 2018 16:19:13 +0200 Subject: [PATCH 0160/1853] Removed obsolete firebug-images --- docs/topics/_images/firebug1.png | Bin 44391 -> 0 bytes docs/topics/_images/firebug2.png | Bin 69392 -> 0 bytes docs/topics/_images/firebug3.png | Bin 89644 -> 0 bytes 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/topics/_images/firebug1.png delete mode 100644 docs/topics/_images/firebug2.png delete mode 100644 docs/topics/_images/firebug3.png diff --git a/docs/topics/_images/firebug1.png b/docs/topics/_images/firebug1.png deleted file mode 100644 index e2eaefa838e627c3784ee76c5c8b13dd60783e59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44391 zcmb@u1yo$y)+G!f0RkaN2m}uSg1dWy1QIN`OVB{!u0;qIAV6?;3oeCgfS^TTg}b}E z74#wZe%*cFd;iznWBg|nqsBRPZ127HT64`cH$flerLZuFF_4guu-;2cC?O%CL?9tO z&_I6x9C3Vk`V#m;c2JTMLn`Sf-2`?X+e&LXAR%EsyZ=K*N=$wM97J<`FDHpM@dWqD zODsgYv>_7GOQiP_Z&h5Uc4pl?pPAgE9ni`plAF_8y=VQv8Wvkknp5fDMjjlT%o38U zgc~jX`;jJ&=9{;;pWa^32cp-e>IAcpqgQMrEAdmCyQiIGOATLl59u6V8%r0 z*7-lRXWEIG7oT;dm1|1NTz$*8$u@BR#My|+vKH1hN+CXBbBezE$5PfPL>b)n4841} z_l`9qaKHFXhY%ys$NdSJoKa!&M~J-AW0L;GeZQ8=?WA|T-9^Rbg__RpniF^akLPiM zUA~!lvQyQt#N||fb9v>O;-w`uT zF&s^7v2Si2n8!hFB0K5JH|tG=%5u621c@>UC)?7s8Wo~( zLSyvPL{}&9!;8r?i!*`ScyO#|Pm)d?7A>FPjWd_9UmI2DS@V8GgBIrSk;ROy0mOls8qhy=RHN z+2)R!6@WV$-Ep5K-Pjb78FZ1GQartW+WP9uxgs#M%GCBT%9(A|zQl$KCoy*@<3P#x z8IsMj+$1PLTz69htLI(YXm6p;gGgRslgl23vjw~-NJyqc@47ahfzqU{-CUtpqD8cR zDphJUzNUc3h@K`C8 zLZ=MO4kYkya2@(4iIzvYqSVqsa8-Z#Ay!(fNsO5%y{L}q@MES}KAY|a?NLMh21Cz!9$fBY;Luq`8J0fG8_tuUouAhtSQHa04=cnrc z=OnMToD?0@#Ldq8XM1t8mfI}FtXq>!EvE_Jwx3eO+f`~(Hi5!Y1~}4ln?};DFW^$% zy-LPTTWp%{wac~>cIaf2mS%Zv0drgZ?Z5V(wN{c?@fh8hEf465b_wIleM$u^Ux0Rm z@uW}LOl4aY3q&$TT}tUk8cCX|-gI?b7*iax-C4q@shBzYv)H3<@7D^7)l{{UXbD7Z zy7s`%563;TLz-xSAm!BlP(a%;HdP}tEYz?HH{fFbl7^tlocw!7pGHNGJgVn z4Z?EWT_4B)AZg#SvSTvY_ZlN-sVYsMlle@12-|EuMfFzvebc+4($XaGq)KUt!cJ2B zrDv{l7SHizpI`bzNH43XO9D&j?9Zl?9U8H2sfutu=iEs~f#dVm5o}k#^){DF`?WZo zD?jVJN#sA)#>Z)D`bFv&U}rz{?dE~1!)Es zq3?f#J1EqYfK-ds5khzC^uD={?=9ySDIE(^m_8X56Op!mq0!kD!Ihn~>>QvGeI~}j zRBPZmntl_gRIO4Cx+%f+q2g%5=5WqqPfI=9;b3^y_0avH8ltJ!5aq%A(<1Z2Pm_Kn$43rvQCF{> zX4rs!(`eCfe;@k_)a@Z2U!uGf4Rq~^>Yn;@*mHrbFcvCXybYeMTF2ZD#S3ag)FXXX zoFjjI*p)@wm&ZC+zo(F=lD`;lM#0v3ZLUn<$5d(aqUNHM#?u{E+hCmIJ_WXPm~Wuh#f2F?Z!;(7>9ws&fA@&zEv)J}Q5im&c6<;*;Lw>rM= zfgzmLG31l#^WH1Uw4)zmpRC`c@sEQ{;jEwE&}NlnN%g~7Gi}gPE$)!hswy6{IDU57 zan)hUSfMH~dhS1eU3=LWe~}rUG_ctQXGmetju8Isjh<3wU+SGc@Jl<~3$4OZ8a-cm z)Naz)cu+cok|oNWKeI#=0hnxI1@E5psG9I7~s%Qj#hMhnnTt}LH~Kc;SchTJ)uz^W{H=t)w;XD8 z{S@2BL$J;-wBsDi)Amd4e$X8Yh=4I3j>}cAQhUygKUz7O%qrM&ca9(dPaX6fm|2oj ztpRa19V$!Cm4Fj@q#9iGE|+&}y3)#=+q{8wzxvGSD2X3*!FQFTrAC#NhfVt={zp!C zX3JdvOkZ!)t>ipYZ$4~@{MeH>mt%!=cT4Zz{h=?_DqVNXz$)q3_iw^`+4xM!R=507DS4I^YuPPG((C zrWeQ>q33po**tHNyYqBc_dib|eQ@qP^@Jn@Rz`+e-S(GhNZ-D^OkXjO@sKf`|4o~j z8qa6@iajC45*vcn{jJf*MBUD4i^!s&(6;_N@1GEv>vU#vOKqQwpZI8 zRrM~#I6;qQT=fw|(S69`p4I%x$~W&|O(+U>#oWB$*5ED^C*AHIs5#?du58|x@$kx5 z5ze1CPl%AWF)Ze_1mh;%Vm&;|t#HiUG+HDPgDV8f6;@2r(#X*q?sxH85Z7U2bXm&+ zuR0tmQ}RHaQeWfroFM}8jpkBDu)^t?0?JT@B(`vsWS z*~hylz}93zPy=u*&H^rojhURQ_FZ`iOYN;8*Sxa_Mx>m+08ii-4W$08-6|L0fxyR}_@cfOC1o-X=&-<$`AoA~`WJPs6XIK_2D zcSIpq5JS)S=^6$_-plJYb9Lpzda)n?ianpd+uPsiYnWb-o4pTq!V2RhEf@Pbgf3=; zF6$wAM~=JhVI=)K4GyOBDI0H)kQ%D0h)g&}>z!g)n^YFNcO`(NMR>pW+`l*W>~|L$ z^>>PN%P}2cU$NVSD@45^t*r&qYopNo$bW4`nw6`mSgJ*6^x^yRvDpbLUd>3sn@ zp|~ZML%-bAh~3mOMGZ+p=&fB*{147DLYPlw3N3M9}3ieA9Mq*+mg@q&2 z+B`<@?9B7eM)F^~2WceV*0iWr)+qPA=H{+9f%dkyarlenYb1*RCs~)EBAy6;9aw*a zhN_e@@l&EmO1e_qnnvYuD02iLC-H3rg^=lrna?F<>Yh)yzM}$ zmAqUivA_rx+0)rRgY<$zr|Yv(Jh7#vnS@myl&T_GK~q_gXY2X-R=~%_m0T`a}_b!XB=3UT)a4J>0ON9Mg##J^mXh}C4DWX;(Q7rHbb z3LaoohfmT*`3}p(=V|7z`u%|E!$cGh&lN({b}j~PE|bVHD3-!YP(wBzlfy;fRi#a8 zIUhHoqJ%3TMwL^zlauT{J>(evw^ic>vLT`5%)QW%Aq%OUX+kmd{L=AoVjAFpV*b*c zTOI}b<;dx%dLaX&)Arb-=;$ZFc_X7(g0?TSi($3ax-^{+>P3$pJuq(pCrnq5o@`RN zUhf*17Q^1mR^}Hp>!roh_4a)qvyd`2eMd|Et!Ou7*$tnVLCC>0{loL|K{ihqvA0-v zhPYD=vCv=1o52cFy41&aTmFiHTMXKB0>Y(t2S{Slidv_luPJcR&Zj)gaR-8=Ze6f{ zCCv+Sv#|_XI=IJ7KHTaRy)E2w_e~09b4|Cg@gDkB`)eg%5lpGKVYP9Q2qQ>V5u&7K zW6R2Y7Z>%6)3uDTNr+bJ*|R9J7aKaN+|l{eR8;QWIJ{!g+DSuj&QP3JQIgmsYwys9 zw&^oI_AcZ4Ql{eI;YTa#7H`)IDky+uptesiPE=N|uQ0&Z%=UxnNp{wpA2*e$I5`D) z&eqmZG&Q#<#qHxAx=vhB_)?`>Zi)Gwi_mn_b2=7QX_k-O+!5$v-;~UMHz6YpMsa|d z3kxkN{fd%K5;l*fNSG4{p@FyT(VD~FpOLx*^j*!`TDjMI?LnCEg2$9CY0LuUR8Bo% zUAID<1Y}Rg?3F>K8%|E{V%@)dSo-j;?5uXJTC8!OIUJizw=b`u#g~`!>L)#VKu+mb z$2?#u{ti@#Y=k^ZdWkQ&wRIwTGk7#C^Bgc_-npN*Cve*^pATuoii#zD1>Mr>pz9@9 za==PB>c;*tm)wrmT@=BW+oqim`#E*}>ZICgZD zFD|ld=xt~Uk>iFuI$pax+24;$PIlQ+iBgk!>@URP!$5!uOf5;tUD>I`w9>lvWTXxARuf1UU{I@?(gY&tpO8Iuo@nIXxZY1vR-Vqxf+YBR zU7_>XJgq2g8`sCa8t~QvjxR0!$o`6wz}3S@C5}Lx!@@Jf4dO)S4Je z%!ke{0>OTRz0bR3i|3-|!7qcmjKb96s=IBx1#79>ItOM`84k`-m(3Nidn-mIqJAm( zu-1j@4)Fgq^RfEPMI#pZzuv5~8wsfuA5riRBUq|P{{lFUQ+|*2p1pRQZvnNsl-<9} zuOZ66XvBN)GRRDNEF+1;$A}D^{!94yXrau?$mWHf9rBL?f$l{B=6oS<#_#-XoOH|U z9Qxm$^AK*86immb*xXn6Y23-MHJzvu8y#p6l)I&9>#bXPY3Z*H|F~#s0XL4e=4F$8VMv}(>PpkO1=Xtx9i8m*a(+VA zYOfPTMEYAz$CQKwi!-0KL+S<>==CYo(LQnlca%vZ>`I>4vU~~|rJ@32V|{Vhb$a5v zV|n@oS(6DdW96l!Gb7?;5s_z>SXk)T<|pPaHlJU@Msg*3%YJuph`MI(R>wm}msK}8 z=-cYOCJU+Iz3U)EczQiD8{kP|F$*#V%?@iadwCaT7Hc`H zMHTTgd$`s9@*^*AUKNOOb5%i9x8BXc00f%x$Bc+$W!x{b8Y|m3f#(+NSUPD+lX*S;+TU*>Rga~G7`T~wMk5wfzh1RCylQD~dNDumKggy;GL#=P&Q?Qd_=J3AL0 z)~{rfHBJCmB6`dBGvEB;JJDPSVcaC?2BUhk(xyQc`&^hV#DITG(7dd}qAPsgX;F-X za-|b%H-uGfaB}_f*eYCfE+i+dp;#eTmR_rFSU=;%>p1L??~b}v1)0japG*-~zsB`T zO>;T*y_?)SFhp5d!PuyGw}>ie+%>LPEc3JdX?5tKz^`Fu-3~`m*sDNU1x`Y8qMMblYg@c3N8-=(LwYl%M_tWG>_r_U`2FTqcB|YLQL){F{MwayAt6E!$E-x(v z-Ge^uZcJd$n*!E3z4T`A(bl%piBZn*OC0IFjDu(B)Q*9UBa?qwqMuG-VS{SWq;jr& ztWQbN2*;>1PXY7oq|jeI)N)&IH&(QbFKIQc#!LWxN>-3VXR+3|J?{zsJv=ZwNK1xE z7lT@^F_vGl;Tj_?-7RdNotpXc>jODk+W;l=jn?iKur3t=bi@KhQqiL7EmQ1sLQt)g zeV6qjxebp37Tk_2gcR^hgU9@Z?~I;2krj2X6k%i2&0l`S zPa1@bR#y{?iE+>c6LU~sH|!@yMjg`yV?6Hp4C3SyOUur!4kJsV#d|;LV(joO0WD9S z``tU!&C^bPhv0cH^*KmHi@TG*CPSU6+@m<@>M`b&E$zL$0L_(q|C&0WNgwRKU2Ng= zrou|wscW;TtxbC$;``NFzVPNi{CH!i4qq~qIEFEXL9pL(t~R5pS~)91z^k7ADM^Y@ja3C0x6TDW0tJCd;=D{8~a^ZrWY4_mYITp&^T;Vo*>W&;XH>kIlX(?*Rh@dYOxRGxS|S zo#y;x`S!)H&)ES_dh{T8;?GGk>_nYA(5}tgm|iz6n&&S!SDxn1P4z-yf#+Dg~tkWI%-`?+uD682-mHv zM~8Ek0AF3AZ!^A~-zR&zGud%+#P9II-kwWDQe34y>97r$`^NFsWm`Yv=x|XUIGOWRl zDgA^_qX&*wpZCFvgy(22xL3D&9h{sP=*?@K<1A=foFv9dv1iJ~n?H%Cf9&3Fg|A#3 zc4kv6YQ+E0#3SG1u+QH39rP|~968o{Q>C5nPd5RXlo@RTZty z=^QM}sx08c(GPqiG6og#Y2vmYJ_wS{wf`You9KmnB85nK3VP(c*o z$FuNl@NOfj!D4G9GfQfFANpx46=hLQK@)rwC48ZShg6?Nb#qPzYO$M5v2yU^aU8&< zyunf2h$0u2tH1ftP;B$ab@U;UF;Cbe&&@caBkR-4+l^=B3U~ysh3sCvVplv_@OvA# zkNy{Snq%e@xUs&+Oi2G-<@Ns;F6h52^qylQ*CQdh>uZhI7BsLGNCU8D;x{@;2YfKqAmn5`4K1#nlf_CaQ;BRgg zoH_SSC(XCCK)>PcU7z{$9Z5dFr=D<%V+%^gRMofpQ?2M(vGpLG6kZQzUkhp@4%#09 zv$ZR1VYCp}!*>2^2NDn+jkjiQRoq;y=6zWaY1BWZd=KabjOI#7o~XFhIj_rK0ud14 zAL^;iIv(QLMkM5u^OmHS$lvWXAoM(k_HQxC;YO{2*x0pJ(-!EI`_0ofx1Rdy>Uk9$ zy9%OJo)@2*eDziAMDJfrB#n@Kufj-*jZ%VToV~P^OmvJ(cVgnRMqR}#4vuLUGps|B zm?#0!hPkR<${-kQ*sSi6{8(9!V zFE$6HyE3ugSoOm=n6s3KYOITc!Us-IWF4GhfDm#AN9!ih&g$zpKR>IWcJU}N1Txyw<2JMWBSj6==$Ie+1B8p4Kx#B<>S0Ol+O^8F3{(n} zV4eI+=-@lu&1EX1k{Vw$GpkuGujPPP`Z7;9ezMR!1>UM^Vikt=0nr-S1f>af@SYI$ z#JP!^`l$Viqqdu+^!tR4Pp)qY{=)j+c5&Q9O; zdCPS;#XM9i{t>F_$jFAhy&)>LbI$nM%4b8v3jWSdQXg{c00>>$Y*FxYaLbX`<}nVs ztH3NeOSuI^*trif^*;yD_h;$2_r)hfs&Md;J#*9V+1bsJ{M-AS8wZaxSy3~;!IPhX z{{3%q(LdwJQA`2&i4)J^UyaU6`1vJcyK~I*b;=^HWgvO^C%Tpjet7wV zgG8?`Q>U?(iE4RFO6CFb{$R$SS?Xa`JY?&X>cW#8B1N00{U|vl$6eT@JYVOmqnUt| z=cf50)2=@mPb4jw5BD&x<)r#W+5?{vgLKDw+y||&g`rKlj2|FUXD)B>CT-e?b_iF} zpEj)&{jZVx$VI%|a;ti`U1|lSZ@j!Ds95%}f$&eLT7_hbJAFpsX@b~)W>@&W0rD5| zRUB%%?n}VL(J9AEYvvo}`Ej!%=wfR2CR#i%tF@7$A|*CAhhk#H2hzfU5MemUQLd+8 zHSrO^8GSiXb6NDFqR5olwS2LeOR(G8ok9pRG8(&4zkN8(F{oH%%90^TKF}vVHjh)eR%HPx1YvG3gP5qcs zyIKWk`H00b;1lTSb8@w5XxGVvG{%PwrDYwxmxAV=(YoV`S%6NRxeShsS;eG3e8`v0 z9p28b<}uPrv^clE?2f3@IUDt7_Oa$M3mXWe2p|eo`S77jJ ze?|wEDQV(@7kHxZ@SB9EnVFuV;`P@@IVl!pu(O%aiyF;eual|53^oRnVM&DpX*R{h zqdTKhH@f4TMy@#X$28=Q0U1;aUw2c)&Ce%=wS{lyBnIrCtK<;=@ zL=k8l)pB|)4yVG+;WSrHRn1#kqRxbD52zH2x+RvKqlpyQeg%KtXkf0ftP@nex~RU1hiB-x-2$?#aB0}K}( zlPn$1f&2NI=$S?-JkQ3%t-+DrVklDtaoxZtc&wx28WB-asD4esk7wOAhd`v1>V~*q zu73GRr%o7`RK=|Ms@ah~7}{K4Ml(o7z_R18@P>}kc(pgUSQ{j(?jP8Yo%>EMd4mRz zzpjp-h9*elPH0Jf>7eOcY;E9tcQj;wsf|QrrU_za2oZLny0s!Gdki$Pmr_jI=gWy3E9XK9(L)}F=!h#IwgE&!1Q&=bi!c3Udmf`*;1??X{I zfKP$V*70J#7-`T*!}>|>-2yEf7yQNHVMI89(GKt)eK|m>O{1a!IL-T0|E=iuOL4)2 zbHnP(^{H$fZGSDfB{gu$ukFwkk%>o0@&dIjfvA9YnYkO6I%{&=t@-RG6tp=%JMri! zfF(LEFF^!R4@C2j5H3o}^Oc^?>x3+{#N)SB>pkidQ z^xnw$Dn-7j`Vj{{j924g(|x!ciY;yYichmIQMCxJD2m{c&I*DKgr^9OuZQv$>_ZLM zh?kF#jk2yCC*tIo2;%qOqX;)-FxrbDP*S9EW7Jte~<~lkhvbmXYH8skZ1ku3~ z=E%rIg3jG?&?GB)Q9g%UejXkvy66CZ6LJxqt|E>UjivvM@`<+wv3=YgQ@y&(X=uoU zrRvU1S(K^ekx6xd<2yQb0Rq9zJ-4p@1FbBb3f;-hs*`}lXaR|#U42c>&Th>r@b^B8 zokdG!ZI$NuD3V!ECwJ&Yg6iFkS!j!Qy>xW7?I&{&!BJJRrI$Qb1zHU}3zzF1&CL%0 z(o-pNW2FcEOl0wpdcR4X1UF)J=~ZYbtj@K3H!FM#4XZa3jgC_ph7Xx=B3Q0A`ldzl z0W!E7=~MQ?=!`pmS&hjDM!raTosgrd^RzcH&D(mmS(r~~#(!WvsN-t_XOs7(2@W)0 z3ccJFe=PIc>i6w4OxDBWXO=TB;!}!wNQjd{E7jf<^Yo{AA7>&4yvGKg-yViv0x2om zbZi_5h?R*+R{B{tHxKt4;rlan1i*+cy!lsxD31`}r8Kn*S?P%)Q$;TF(zGnqj!xh? zNxW#))t$PvIBo!jz1h!Vfdu9JO$!}X=jQ?wjv6mUML>gvVry-!(%CAfN}%C}E$`oG zps!6gfH#_)+^Vc>rJ)hGo*AsDna{-{DF7-2{{H2U9^>^P4~RG$0jA4n zUjTIc-juSJBRV%@S`}r8y>kbo)K#qUAW685|BO?nu$DBM+o;o(Nl(XlGg7IT$JsLt zemDYgzVBU{`g(nFuGGJ`V26F6s&wW`t*#o?TEOPEKz=HuCmb2qGDuAUDNEupUh!x2%Y~X%n zs*r$OG3GS3G-g>>mBEEMD;Ol3#BQ!1)mLr!*fPmueRzPWc**|b7#Gi$zT4_MS=s8s zLN01*qmJOC090XUzL}NFUtO}>2D)0`O^J=kSpsVsunyMPT%B(7#t{--C=TqhMa z!fEEGWWHeUq(tXzTD=Jf*KNJ{LaYTjlg5tP}N0QD1PLjO<>i}?O7_Pnd~3** z5-%6>Rd_|s3QAj?*yW^gypCaCrlvl5oSkqY(qlt_+64)307sntTECBAepFThXGk3D zd0`PLK;HB6V&~CK%Q3T9uTOxJZV)z3_I7p+X=z_HJD6CRN3X6i>x;$fyK3!H^*x6s zCPfyz4aYS@w66my1ku`>H?}MdToejxmk*^?qPGS2Pa149Vpd1CcM_b|herJyn=S!N zadVbb0m(yx0yu9L1Dty^lHUYmYJjXEj^#ccuf?kZFChxB_4D&Y_inwO-@#L@xZr2; zC?oz=45JO~;S-XzYKJ_X91fts9?QbaORpapl?A}n;8%T?Qu}i^H=dy|FJ@f^y~f&| zt5fHu)ZRXofmGc-C0h0-=vG+%Xmlat!02dxc6N%)5xaNuKEOWD)Mi*&a5SDMJ@UA= z;&B>dN{eA+#K&k7%g`^+G(5AUFnvCISygovr&ka!INsI#lwB7fwZbPV>Hme)`mi<) zXYy)`=qP~@)ckQCHECon>>RtG$Vs&nteyU)2?h`hGN$f*>(IKM#@o9p;OhhA4$Vg& zR(~qKid8O!m=VNnY*j<@>n6(_%;b!-G0&k7JbVB^aeI4C0+eCv1)4U=`h>^$KPFqM-OyrF;$+7O9s%`~pmO=1UD? zDFoFhh!SXUq=6&`kS633T2CsiE>c$2vHJQ<)mI5{akWos;*lhI-MFI2q!6;|k#d?o zf#L#s;yc8C=eKXNBKmlEk%xN?_DQQ%TFb%DS4BiPZ1yX48!RTKZ6Ns!F`pLOloSTI z6b35&F=bR4P?Qa>&n!em6B-&EZ*TCbrVOM4KjKT?6JL4%f%t-nd}>imY}zjBDAxQF zG+eY=Hc4$|H>Go3=BqC!0Sgo09V(LD z$DGZWgDu~U&x$0;w!fi{45QHhHAB$1TJcNoO+Hs&lDe#Z<-8|kYVvYS_ZqNgHd}+- zO}Uknh99p0hf~%r4lrSG1<~0CIV&sfZxX|35b^qB*o#-s%np5pzSY%j9URE60ZE*s zRQTAmS9T+icPoG^(9%R7yg6@hxx{n zfc~dGNQ#(9=i50%QpTYl=|Y3-u2-VUaH(c+!7lX7wm%Jic{hpaJFO!(ABP!2V=Y2E z`J62F&*!mE4{QOv^669CGul^p&m_;*EB;8tfE5;j^H}$A@Sh74{x76b|76_$J9_N@ zybj|2ac{=Wfys2L2HO~No15b>Gs{oYzq&Z|-*3?mN%d(6?ut5ajRYe&R2huARyQbW z!@Vv|mgCtk$;r)6Xk1P<8VU92Tz)F28kWry>avb2ey)?66}m5>}K&1>vOq*9xGkCWg1^?#&Q`PFKoc7{WtBvSxsaC`81}8LZ9+Uy3=qByiVg}p@BC0#`8FB-*+rFfx zf4}lEH2bG3B>Ubgon-4atk~D>MM$|L);V5ds=BqyoW!10TdCr)rShyvf=T{vdY)lw znjU@xNQ(fDiWM)%%{5ytQ3XKd?1g2^Nnq2AO%zDmf09o5Ct_2S$Vhje9!O`U@$e7> z%>TFVea;Rt4t5bhg{0*`>~gg5gM>7%>lQslOmD5T6^#th*{9HN-GDW1Zdw2fMU1e3 z+UP_%>Xo^BePUOXgoN>s0{htDG6n{h-w(hKyPqy$`ZxNZ=*BYP>Nk^`sOhqp8lSpn=24-do?pvXm2G@ z`M~g>bIeFfTi6+jFsd7+v;cOpNx zG*Frv3qHif$7^fmbLy@xE=DI3b36Vdfh<;-EFrEzay4Y4%?AKB4dKK>k8B7e59{sC z02q5KtIm-TX?N%O9r5=`?e(7(CxPr54Lg8EYa|ed!$z+sf(i>LUy}0U$)oc-{Z3&%2$vk6b~)cfG-`o#rBBgNW; zAC)*H3dt1Jd3S#K{HW$xu6L?|Eg}xYp_34lm|i~9X&@P(@zbfrF=zYD7-Pqwc~YX= zO7*UmO053v#~;T)2=V&?#MY0|{ttKpoo4s|&7!cn1y>}*OuMo1!fc@W#S?sp=S;1h z? zRrG-qR+Sd2Wr>f0Vx#ZOEPTWw>%;T)HPcgOM+1X*a-l4=w5A6J>hAC5!(P9BEyy-l zS71SROd#}sQe7>MX*-``T;5ZEnajUXu3hlI{>A8Fx5A_5hdGibw~kinAlhA-nH9&! z+^MNI8=uDk0F+HV1Y`k#n&{h)hF~Iz;@IF&7z>l--9}a!P|$**j<1z5;9q_DazZIS zGYrr2zAm=vi@!+`^i?lye(U{(%~C_&G*^p-;xz##+t|WOVCX2o8ZBnwnT~;YsS%6B9{sVm@D_(AnqC9NzITWLu}%X3_Rpa)9u%szwd299N`| z-&ci;J|z(tDXto|T@5}O$uKA#dYOBbHyGU4>_rJN zyLNRuRApEODsj0c@4^WZn#DPr@a!9z=G~B$@5LaratXsp_heDk(B~@IG?52l(w{yF zLj1t8tSwIb+qS}hsO>fX&&#Q>=Ny{;QP0?kgOuC~<>FW!u7bY<2opz1hZH))Y2bYP zrYm45^8bSh3_{=J0r)hz)E!mbVh^B@D;RKZuD(8z(kcL1EPC`U?_;;ePPyK^*Emy5 z0Z<~SV@CoS(=j?~KFM)dZnT(?Fy!}Rc-R@F&Oknp3gv*B#2O*rXR>sFoag-b+0A(i zbGV3>?X72_k~tvtSz0;^7b&3Ne4WN}JYm#?P8mWFv$|$+;?BUU4so?3g3MUoA6c1E zTf8{rWqk6K08oKj=386y{2c8BvJxew4=^4=3^zZVgq>=lrJg9@7647ik?^53572^{ zn-|PBtWrs>Rq(#%=igswt&b%TH_W_#rf`zVHx%O4jJy{x+i+c1d*HmQ6wkcQQRJxR z4hY_4a}(KGLvT1giIo;Ho%Te9u6Xk{8Wy#_eL|K90T{c>mZsF?)}nCFFc!^7)>xjD~#uEY}C<24J=Jc_{DXcmiW^hNmnJ4&S>tujP++78GRo`1IS?v9PhV zcXYfj%_Alb7@{Z!8e#XUB`Zs+eZB9n+&PCsxBIZ7&pu&Q0|}tJzMv5C4+}FguEBV( zU%w6pBGU1tP`SpzMpn~PQuGW=KmW(xcU}i@g=^G@e3%a#zaXtg}x4T5s<3r>fI_{xg`SjYe|;qU+W(E~jobpUjIPFIy~BIRBlY{F!q-ud~An;sA9yX?(E;)C^5Oh2uTepVpoFK0|r| zsvUSWzt$`;`U0@#gI`TzMybznkiK;9PePgQm(~0L_&2^Qld-UEwq`)B4s2mh0Vo4( z_wa8+@4Ppza8DA@CI zH@esDEG~i$0Il&K=)f4Ss6JV4MELvt@SE>lQNMp1D~}Y@;7bB>#9a}t28bKE!qLUD ze=f5d66Zpr5bR`g+{+8DtCI~70Vgd@PJU5wT~YCfS&DrJ0Gwey93~T75uWIm#}WXz z%r9soMVcJ~WT$WiU#`U3o12L`2d_ETtVFM8p`9Cmu0Bnm6{xTP>XC#zeeYRnywg6$ ztnhGyd65njl2D*JfGw$Q`M>pr$YkUTL_cgM+z1brG%BCb!`~5g`wrKsT6hX)Q z@}+MA=L@!%jDp&7qPG{z?pJS}tKENnVcO#+c`^bdyI}C<-kv$2Bd@6@=BrNbiX!Fv zNJ0=xkN;tL$=-zHIj7<$0R^LCFvL`&2sX7wc5Vm*N?_LWo@S=S|m)7wMAx5~>=b4xyBiUCS&Y;|M`qi=XWk!=P|S9g}jR z9WU^jtb?Ux*he}*Mfh!>pq*H77qQG{h9U{-TCp+5iU7Z$KU| zbk{gBuD-C;Ci4L=MTqFrSfQzcVCrj~wxH#@o|25WtOR+yJjAkb7uWaSd$uhpTa|)# z{~~{L+0ldNkxiQ);tl!xT3?AIOyUBAYpbK2i(jm>S^Duw6f{usU44m`YHLxwy(E3i zQy${w$CM^9bsHT^u&C=zON~1M+yU!w!tAnB|+x>Mn}?j;6JL zV-{)7JMREhorQ&p(ei;g+?`cVNB;J_wR}?uO9qh(E7M4-f(h<*BmxRgK%U#+_P zNdaQ{Z{j->HOQo%dt=Qmk5_$-SjP2k4Nra4_V%eaqL3FdrWM5$K-x7}mFr@2>w^Ng zd>R2Vlj>eW&WY=r%{&=io2Qmmp$)c_M@J?!%A@|6RtX900H3*Hk0>lvo3t$MI3ish z(;{KII_w$)viS}xKYrw^X0F4kg-B@F&VdJC- zh^bshy%TRAftDqzN80&zT)A0zV`k>8C95ptFAkGOfh%N{zq7c5M}rDS8W8h&tUC%p z`{bXoEdD;6F>aAn#E(fTeEPU2eYyxv1e6SVd#Cn20So17&}}Kf4KcB(^2wyDntDk9 z4G!Vv@tAh3tCEq3!>->9`y-`YFJi(}R@UwcgFOq%KG}X4Zg#oVvo3RTl|(3#(l1>K z@CAN;#vk#yjit|A=DC4_pd$WryvU(3(*-@qo0sff8HT}jJ(_h)HEJzYFj%Vn%*^Yo zh+deg=()}&RXD!Jj#5ZzX>Pbjc1;7r4WObmFt{4ZI`#3{Bo2*`!Pb7l&yoS+w4a&| zJv`(Gl-d@M`#F4SF&$0s-`rdQ5HxEkD3s$&PZ^;|noI-i2Tawi#NKx&1q@bZ4r;N6 zDZ+IV4r(D#poAd=ZuH6gY1Fl~-p{-jiEu7}?^NIG=LnijOb5=&-I#!G%7qpSuPw!>KLE<~ zx%172Hj!07mhtL&rU}S)>mBniItUpp?|W*!kG`j8Xu}a-61VAg46DKs0s@@^E63Pf zO2T1dT7Xel7gx0!Vp~q(S6XSqT78|cu&&OWx3*MQqZH#J{N2ikp_z3$Xxg^3vMt)8i)}f zt?tPux=9i3vdW=dg z&t$V^uJ$qEaj|0T0i<Yp_10^h*YWE7Ms71pcKuy{L|p+%bq z0v6lMs9x_{=1tmRd?!sfMx3vuO?e;7DT|4*$k-%R!KY|43sGm6?)?ywpsMg+Y3t{`d9tGyf zFN7V2ql1mhVmc-c7Lx5<4i#TATJiDzCizBnEI`xA?-oKjJ4VF4p!g4v{j(M?8Kh{l;7g$Zs)UL7_5DIS_&WkH#r|p zWHhz?0pFhmpyuN&ZlJ7B%H1;e$UiJ1OK*({A_lXjA*J9Uz z67}~!@S^`wp?b`V((4y#1DH<u|Fdr=OVl$B&(Wuq-yl zJfX^Qk@J7B_timhciWaE`AC2eEVu@D3l=Zx5g6T{bV4IO9;dEe5-Sl7_-;T&4O5ef3YiH3Sk zYC!(^2lBZ2Z^$F{o6FNCkms{0Q@-xlVa!UACB@WmE4N29OaZcGWR$3yI_<_&{_mM- zFK)8M3Er6JvP(a#|6<T$SK^3=}Mv{1rnT6cbpu?p(uXq6ML z)5r&)|66(?F6GIOsw#sxMLqyA_r%RDwH-lEO;k$n@_+oKFfklv7g*E!TFg1aWUel< z<>jDV6w%*fy_Qi;`3^o<*H53v@UgcWYGl;oG&D~)02)ZUJO%jfAe4iTRXL=5FnVgE zVQ-YIm-)2rAT%)?0&2moXO=1K^vhEq+LD)_D%UeJOxpQ*2bFr9L76S%;WJ?cv}*SS ziTq#BnPK zH>9~%c%G(z!gQkbFDoYz6&*&S2FPi#iM>BgeKH4pFN>IktEKqEtrJsPGCgI`I!;p~7!RbT1(+U>fDG=jy7VfT*~ZLbQvZl0ciRfZqpb z%5xQrhmY|jZtoLkg{|&cU(){KqWfxVLf=cJ?v|D+?i8UxY*RDCIBlo|noF#a9ei%l zNKfD4+Wl8M)hEEm8k@4Ia+dN$Y7B#<_TznXze82gw#|l@s(aFAbrRfL!z#@ehqxo- zn+TAb>q2<+mYqeZ-?X54#29`hw{94-C^s-oHZ)91#(C;E4?od6G%Ga>yX_{WMWgO9 zEOn$hI3g%8+D=KKDs?dK*qRhGGV%ts3vW-<_W1?8V{*ya_)LWhw17HV8PaHZTA;`h zwi$7IUh%x0UvT!jSYd~nL7mh+bB-xOqc70}-P!8k?4jw4`ua;kS}P+9(6+S$MPyz1 zOux!=G-i&_OpiC`GDH9;%FLG0*1{6?6WebI`V1y3RJ7bglD=)zw{;2khYpuRa=!oq zGj&NRKl7RNm&V3-IFT_~rnv}Sd_k>l`qjy!4caIE_=FWa3hV8M&R^`7_HZ6z69UFH z7^*!qz|uncv4p(BwBR2@B}R^&UI_QcuM7{Ff=Fs%-`w0N`Iew{n4y_}36OTYAQmyP z6L4zX-Q_|Qnd9mf0$an6#c6O6lX2P_BMq-#Y$R>D{x8XeuJ4zZ>p-G-ol<`h>X3)L zLGiVZEOf1O9U79@uyB1#X=hTpEh8LZ6)Xc@j(irj3GCz7i*7n-fVrFM&^;)vaj=+XM_Q@PSKZ{H#eqa0w zmsy}_yUm?L2D3IDY%+_{8O0Rg@eIqdp1391+vZs&v4*tFfSAs8h1htIeN;6++2UOV zZJmH28Qhn#koIW{jha;p-_pDjAA^Bap~BhxM;`fkiAnR*N?T@ek3u;%Ehdp1P^fBg(7hhT&IhXsfYC&h-A?ds2KC5ROC;%+GEuwysW=KAbR?a%=G=^4srB z_qniftDo94VA_f7l0W;#Ehr@gpLyhdxh(2#%N45akbcBgGj7c3wU!h-nr{dKGyN2+ z!JE4_Zb|&z6bJs{pigl`gSmaje$MtwaM#dV%cD)kDqqoWqKpsxV5wm;zB1yMT#kb@ zj6NS^Zi;t#N8C_LZBQ5XR|pjplB)#++(4(F#ZnuPB=T?A3zX6MHqaK>zNX&S_kr82 zz}Tx-l$rS(ywvODU3Ri_Md=b?yMdCg&&B<@PRiMt9z47cV?`d=7$s%JscdXHKc9E6 zTg7(APCMi|pE(Z%6I@+hvi0;#83UhIDLlh`h`&`bu-hoGPQtFN{2^caV}<<@CN`0t z{sH*#4>P+^%A?vp*T2%?WPktZ+C)H6(b>`FFNIj4+S&y4zoKhvCnM?m@)GZzmlQP~ z>sAdgKl!P~w7mDWrBjZ}Jb3np*~dwL&6N2;@h#y;Zac3nV)e|QLibD=3;slN@0e>m zOMCk{ld|NTor_V1@1V>|jqheFlK5H%|K6mBJ*Jmh4cm{uiGPC{zl*x6+(0B6v;G^Ri1wY5Z z$j&~sT`^uXk=xI{qneh*{S!sLquhuK`SyCp?#G4p>skS?o_>cUUS^Lmwu_Py6La%s zi{6qH5kUd*#_~^n32kQYgoAQsW~d1XHC$b}LEy}_vU>#<{$lGW8ENT@_$Wo`A{|v* zhZIwXloBwFYz&Kwi``axy!EHwD0F9R3?Jm>;paIZK@n?XJE?Dp9#29~KhC_{UQeG9 za3JeXj54o9e^jm&>8~ZnP09&qM9W(=2R_vv*GS{?Txe>_V#v%B7k1aEs?G$T;57Cv zEr$(GyxM%e(iv9)^{!Yl0<>%3k8IK;PCDBD?V$qpB$e^rJwr=i}D7#8!Z_=rb2MNk9fN#i^s z_I|9fdwFG}-S)xOWY!m=k4R+GYL64>+3{zeDU%x+NUig9v4TQ70v`6xn0@_>gGKtr z>C`bP?+t(pBWMHe+=)Ssy7s0OQhNw#i+(6nU1t&C9a2uVg)X979z`-HJBoRh1bLy$ zm*ce7t4j(O?G@_Q9InQ&Qz|Jfto1)Isva_(`72Lgg1aLr@vnnb!JSE3tx-g2_N@ny zkr8XxN8iDN%gm&54Y-Yb4W$oGcjZ%Fsxa*Y9SeuFg+@k?zG!^4qr=gaqZz9+IP}VJdp=712Hh zs)MVm#}25P51d`hjf|{G$&jS78X98_K74s_J-uk8K5pbIby{sZdK|jEHvLxv4YhS5 zy4g#jE?L|xEDA@i>jPi&wG;TDIS?$jN4hDJ7F`SNq8NrNfzqTYDNM1$NP1^MxKJo* zLOzwpk+Y#NU)cfaE-BzWsfgvWvD@)kVeuvB3FPo~l$K=NyPxqR>PhTyLxXi|D}2U* z-dn@nbJ>O52_8^Gqdu<6!xW z%t+4zrNQ$fdx^Bw2jx-f7b#r$7@ioY-${$YR4#VBeR_o{w(en1?4K?7#4$+RQnkT4 z6+sK8IW~xUxoB2nrEgPi#K1hWfmrpPI-oS^e!}i zygttdUDe(FyKR7N(x#LP{iyd*gsI~x?+Ehu0#GOcYAwP;)V|!r6=1@2 zrQnk^3$io$Jx|u#dq`xAIWYBX479q^HSCnWc{(-pRIdDKMnmNC#@TMk*!z3sM@Ja( z7Nn&3=WH)mR=W;$Ej2|s*zMdrRP1au99<=S(iX?y{P0J4{5{=KZU(8Nd^qtIfmppd zm4zeY0`PL!s*!`b@oOHdB$&xJW8d5ae)d?CZ=F#tOG{)z6}f+mW8B~<`u7qgs?2jz zeCscS1ojA{LZcENud~N0_$6VeA6~kfCz*Uh+Ek}4*jcTWAID_HA^eIwJXH8hv4oG+ zA>mZwb6ggRgw0&vWi09G%U#d4nq*cCL%iWK2H#(4zd;Jw+;5F6lq?whbEX=+yi6WD z1!$PI#=LsPKdq0DLwPJxny|%{SOp>uSJUoHwYH(ilCot2g7QKG-GcmbsV|9Rh3o`h z2sAZ8_@dyJ#F+_;jXgWv;}^et)jjwKrkQV`RmjG{WTnaU?a+0jDd1-AM=M^sLO<{k$J|5tklVp;%7rCe%A@gEXMfS(0|US3uN#r{Hq&p_AG7k{_zomMTYm zNw5R~ui_#sD*JL5%gmP&v9L~UIE$*?%KXp%&ImWC73Q}0Ct9QEIOeEsl0fP_`b%L! z@|q8qAT%T;+fWgQm;6`&^ju*d)}xl^NZc~Yf6T7UNSz@o^34L2*DQ?qU`bL+iq9AF zlA}@OF!qhA|Fp8dB0wK`TyzQu$B#&(`_>q-h{KQf+F><>=^1J(Lzz-qT5^Sj%wS2& zF{*c5XsU8r61qzFyL>ANn%rdsG>%~M4Ez-{?To!WY)&2FUqR!6CP9S*r-z#oE=E)` zG*6agbOuZ>;=r?wY(2jl*P~KOVM9whYgMtcH`Ky&cGF$GJ$Tja-RI!C{q><`x3Oq& zDyvX%gzt@^OY7hR3EAwh-94fFzdK^$KshCtVnPhWE~%x>TR!;Zk(!#C!rQl5&%@W3 zq6mq!Gi~fPwIV{4)UrRd4fH{JeqLYky?-wNMh5N3;y1PfjQ&;Qx`H1ryE$7$jGFeA zZa99s)WH-*!l64t`D3(&<^OJTZdxI2w<#22fBqwOvY1az?LM_=Z(`g{Gd7>JLo$q> z;u+T!W!O0WYMq0$9)d&`fk{`#tdUc#V2b5)f*#X;XMV2s#P5sLxLv>3X3B5wfW7!F z`06f_?l4}KC+YpX1L5He_FQ2w{T(byly6gE&2xi~gXMYCb%W&(kNCvjI=*VpKNUV2 zl+o0$jOl)J*b|3k?7d)S*4!NCr~a_fi01%}n!MHvLG}00GPl|1kMPSsgMpdkW$aC8w%e4F9_zVecyYnX22hfTxsBdwhi3titzMK znbnyAT1>=0bevY?6$}zjjeNELfQ$G0EK!FA8!#W|zIqAz7G{foCE@HhW0&7A507P} z_1DnCvP@`-{t)DKGK(S6(N5(h!0TULA-%kcb#{4h=ex9S!rRJQOFH3N_o^`H16`z< z*$PpSbh){aO-)*HU$u%}>F2*DdIkPOLNXMLQ9UNbIf}umuWR%Vd2w3Qj&UaPvm_pMMMQWQOiqGmd@4I zRtmuTaJ!OB^P$t*x|&{*WqT@TM=uQMG_gCElrAMCs9#?b4i0i?7yf1KKMB^VrS>%) zea7htI?&DR>wawKq^34hVLRD-eKp+E8wHpG$-d_nPEPvSxP`$kqk@84!!<%H$kELA z?@6?@ItdBoz!T28SI*md-FD)9jzz2}>l}#Hky2f=HyN)W&YdycD~eh%ic9#Clwp6N z>gws1`|b7h#gv!L#I!=8@t$wLS@VwW6`&ZdAVWkb*57>`{nGgC2kUovdUYd?n=PKb z+Fh-$6cCp^(yVu`fGD;Bw|cx9?d3o zjlq*aa9NB%FerA^J4xoLzUU8qVT?nW42z^85t&%rYo}mTDoJ=YsFlKpYH9hx)U-G( zOc1nYR@p6g6y&i4KvA`|f0aYF30$S#w#Q6%Bnt8Eg78^*N8^f!NCyaU>?jgxkmR+A zAH~x|7;sq5%%VyL>PWpr`-ELd%JnlbE9ks`HuXt&hpfq)3@#c8V*EF9Xt}wnt~0qT zf)=ch<*wD$k@;VQ3kzD!pY*G%(b1?!M=?GCGz}*hAI5}3=joN*cwsNq==8)5z+y zurN>Wy2h7yM*3tG_DMjfrkgW=GIq|p^s?>IM(PVfK#{7-Wj3He7k6QpAC%dCgd{$+ zH}>6FTBhshcL)iAf8%8qprcz#@?ha@=){p|3oWm9qRoJ_)6mkoZs{b%leX$R1YM)^ zsA|GY!RxPIUug#9v9u;Q1Xs}-EM28Tp{t6Ish2O$o@YKS$)p#dr(gbvT}MM>fAP8P z)~k?>3gvUyD#yhzj6x{{QdF9n{QW^F^ODBXr@jOpqMC&FT-E=oZdj*lw5Ft+@cwD6Xxt{!t^g(XKX&G_s>hzAqASQ^j=0eXuyvo-9 zkzi1bm*Iq$mP~ei$BTSp956K^3mAxwZc#EA}yS^w#=DMA!rw#au zR;W#0r)FfpOVkFZx^hyQZCa#e#>TwM{lBU40kfD3F`=;*D#^1sy}1@BHWmkBbBm2P z?ArA#gsf%uqazv1R_gt5>907Xl?&gv$B-Kg7wlhr^mKEd_*=2Pnq6>A1q!=_b?15d z%F64ODUWoj20G`GIU@$RxhrS6n-K_!r5`N56Nnd1w`WpwiMTn#0*Kd0Mjf<#Wqt7c z5n0#WyH6XgDx~W}Q9eCFsbN(;<QS!-Hb z+TVp|UeVgm∈#%JQ4ej)*a756p8k1%){<#Q|wY!P@P zj;B2Wr^HHC)!SQA;URm}U@Qk;f0F)y=C&j!;NpTjbY-oNFQ^qGGcvbG`})#Fl=`Kc zJSLhlp@J#X+obH%^(LV5Stu!Wty<`fQ+Se-pO%+*efXetHAcXc8?DMW zx)+10R~x)E>LGGj?4uWd7$#0S@%UoSclw|u2xQ|4&CQQGkbgUs$&%J3kfM_bWbjxl zBSMbTh-6h_v$7#)<|CU$wfW-pZ+{5#@=6|!3`X3nCM4hZ$E{KM@fy@kkmKz;1o&S4 zrC;&Zz2;>8b@zxyF3a6X?1}DJeGX&$!JOQPJAZP>PuE{!Ei5!%g=Z$z{v3)-XOd8D z8Hz-%(Y)TpevU@M^>b!2u7B-uFpUJY!~AoX(6PF@o=Jx@kMH>d4~tnN8gJ%)in+P) zD9EDeHO`4LbqqCqC@D_8dygIT-EtUXEn2ly!FS$B5->9U8qI&9UkMHT^bijBBqSV} z_`wfmyPM|4z(en=^8Tg(Wq>P4P=~eC0?d(FeJhlLtj;PT3R+I(N6u>#T@E8qVOmy7 zvG-e$2rqKD$j<+lnB#cSgM1dpiH6zkZiQ07ZF<3~b$D=4%Ebf0?$$PTo33=2x1(yU z0y@O5M55!mwG=y`>gwaD^lQYo)!*eFO6s)OW3yw=!_S2r4_SV0+^IjxewrhhNV~7} zTQnU*0T5dT?5HF~&65bkJ0eo<9OiZi^tx%?g2q|bhE#^AC{^vAaCG0?=3AGWkbRGW zi6!Fd;Yik{F*%Ug*fAFS$#<);jPiQX5$M-^Lu%WwIWC5q7!W@^9CCc>svj(W8%zy| zicfht9sd;N%}@f0=sQD3j!X+iw-ST1yY(k5s2?&;=JI)ysE~$US9hf7^KOPGq$+Rb zvvWWMk9n*ze?2S!TCPl7+p8@5T?@Ado^jMa8FCF6HjF>u@Jn#DBmW>4vAh25sipD@ zRn<+=Q19FGIvC)ctk=I>_+$hCx2z2#VZ~!1gW)>TSR5MsecucCi=)F>VO)X!9Q;;^pw$3R`&Bsoo7H6!qOiw4N6Z>@5{(sf8neV5`K7nS^8pA+vSLl zD>HJkIIU-tPm@W}sH(U)S1vd2Y~d4ZZf--w=z`%tqkN+n5A`kkdMeA6D#{e&#YBtp zKjP+N>3P7**J|8J{}X)Yg72);IOlV5j4=wN2U(50 zwh@gYg^+PLyr;xm01l6u1uX`#otkD5-Erj*hw7>fA)&0g8{f>f{J_w|Lc zSu?7RH;2Ilt;fgA;K8a**!_n0gpCB(wtzyIWoAYV3>3@RI62j2<#S=RT{q?thx^Og zmE8GIelrzw){E+;>OU~nz-X_?YDOMz)x{OqF-Tl$KBSo~Z(>LpQryP)vz#2tKjv zzSE|B$CfT|cbzF8OWvH?7|Il=wB56^!4UTo``ra243)rgSrgLI?Bq1Gu{beO|NeH1 zTV^!B#=YKJJx-^9iG?&ip49u?)vVnD0ab{OpjJ}G;$R_#ZffbbW84q<4_B~$FHn{) za&6^NXYV6+X6xxPxfW3cX<@$xK~d4N94jmHOw$94h_TABu|9}TED3I+bUU?7Q5ja( zK>dl5=?@zI3e!U?vK_hAZ#_Wn@SB+EZfXDhy33OM!q)?&XL@0z?Wzneu`OI+vz(07 z=c|j6G%yyDzzBI52M2RNz|&)T2|QlPVsG!ElG4nrrQl$Gfl3X{@{$sMO+E=c8=K{= zr5M7@=iw7hyX@@pM6vjAFOw7tbGZkYvQgdEfhNn&QvmU)s)BvTuEB{+y$g_A?sJ zqOwHt+_6~_?jeqBCj$+N&08}YPL(wyZX4#|6F5O69q*>(`O!h_UX3T_2gW3@9_-5x zpC2!**R|F5e4ghO!V-jLy1lx;U9U2T<|Wgwt5SzeOGlSpT_YW)S)x-gQLo=qFQ*$P!o>9PiXy7kOHUJKa1@!H z9S2on%sv)R0COKd_}Ey)ux7*?;RCb~I`Lsvc1Z??@c9YtSK;PXIim~BU~>}5Pe>55 zf(0Y77MpxC_dJC+_jWQuPk*O2UfWmNL_2vLfF5?B()_30H zDHUJ}R+r;F5GYejvvI~k*ahq}JU)&c=n7NBiZ$~x4WUuPV8JgwUe8*aXMXUC`I<#K z_4EBI0l^ePj67-l?SHhE&-Eu1dEbiQkM!%Owlibps~L62-}wH1v7)XbC?lo2d4LO7 zVREi_D!!d<0fsv4EaxXcfNi0%yRe3^5-`B*XKglJJ>|xS0ji8mo&zt^9^f4wNL_bD z-#*<71gSG|Ac?3DD|2o@Ew(hL&&IIJPSOzX!Nu;;&Q5L;q4VK#l1CTnhZYY6`?$+E zaxn53u9cchX-U24(sfk#)o+{Q+`)PxsBC{ z#GTvhB)QIk+f4Isx48&jnkNqHv{LA%+mRa+iu0$7j;X++Ol&Zd3BaWq{A2!qBK?GW z50D@lJ3ivmzL>J(HuXOOggxXah(rVhpNB2&j^2Kx2=(K}W$T+F%w(DSeI*64F`nWJv*_*7{x5~V3&cbR9 zDARmL6)bDr^=d0L*t9e>nYD^IMr$mN7r$gy+Lw=xvyK-qkGe>F{bmFZ@vx(8hyh5* zwr6yiBL+YKs+lie^~u^QgI#C74C-dB2}{qD-`nrbd`KkZHl?y8-HxTird6P;UIU!g z3x_BUbmi|$;or6O9+6vJ?3)fu>wOQ zijH!eS}6e-oLc*pIO43G>WDM|B<@rMu*~aB??0GWBr;d6v!yNO;xr>Jjvk97(rNxV z{6#+Of5QHb(;LoWje|Yw{DJzd27S#r0R94;mfA>a4qJafjD zcd1{T>>0)*WU^Roj66&smQYuhzPcuBYeIF<3C+&#u$wdXnPBEKAqKzSF9MP`3j&on zo4NeMm8m?SS}A$=YT97YBcS5Rj6-*Rn$g;->2zx~QL@q9v4}FTK$a^RHC$%BOkYl> z5B)yr^a@!!-B3R>$|uF72u0@79D9&boTTR;7~=h#M@_9KqmUB4^45ZEX5^~7Xj-Ns zbgKg%DX;4vpv?CvB|ZZxR!Z1nr%yn% zYsSB$+t19RdgHZ86Ab0J4J3T1p4V^oPp_ZnTGKT1?(LAw-ZYT-ik?`xJM`~a(n4LA zA3FXSy^jQuBbl}8a58*X@ZlMEEo%XNlKRdwv^Bxobd`AE3>)$oV17b-> zcgxG$81Y@_EgXn~LJS1WTKdJ8DMe8KhEuk&c65`I<3O>K@BVE}x`NlpTS&FS0g45- z&;vgV3eVAgG1J~YeT@`k{RRI6dj9lZ@cttg^v`(b-#L_jKfVE_*E1IEy-H(*BBp+a{ezUkV?J!OqAYwNiW+yBB}ThaOqk(XDXZECi@?~*?| z|L#@D1JV5H`Yooz+&TA)96_Oh?z@?LR z{ir`Ng_+e$Lj5I&aS1X&Q~g7?0)})*C@DYN$aE0mySO|ivUC33KX)B);BPp3H;=wLxzc~Oc11a{8+{2xMrQ`t$V-LlFtTq6(N2oX1-f}n5q5g z>}vk)4JMGdkY+m{5m{ZcnE9x5d!^4A?u~)AWD3d@@td>@nGn?sY}fK$+p-S4QNEEc z=(L%v`!JZmMh)q1DwkLa*zMsjLU)hT?Xz*S#{NW6&Ns4eby{-15owQ)jg@c`;Ny#x znrec-%nuanx~$$@%hcIKT|MwkiGSDQAzajK=tUpN|MmQdO`KCG!wHnrHQla zPsrHMNkVQG$7W%lFwq-t-gk;$x#}Fv&VD9PhhtHQ?aycAzMyUf&7BQ5Kf_2_|GK}u zc{v%i)}OrP8iw$_X#902?am9y_OA)Wdm#b|2?<9V*@_GbjS-&fYsPMx^WTiVC#;Cs zwDIM;$-7YmaLjar3+m8tWq*0~`0Cy#SI-+k3+tQrRE)~Wg!oUr7niLc_=&Vocm)JV znYD{gjE!aG(MHHhhc*;7q@_I+Hqe7YR`Q4f1Ec~l(T%TdU6qvJFH~Fh?D=wKhmYkRDV_aCwaV1>#D8^<$|l!Vi?0Au$k zZfs88o;(-FX&LPD@EOd4;UAs0x7 zCzTp5ZFUQ>=N@8Vb@lY<)w?Ib;k}aT438y%7XD&;N}Juqu0A#OFPNsLeueGy(O`B_ zflnNc^XgW$K;@;!MeTID^vjr7ZSVcbowI{AU`&`;Xxou)1C7dwU#G8HWvF&A!O$HY z2KDas(x{Rr<+iN~sGe~%ZW&KEQv5EF+uKK*KM(?S4RUF(Z2deE3cGlfVxwbLR##y` zEZ#nK?V*&p`B{Y@XlEBqt+b!~T%9(?SJ*e>Gr!CDT`hhoa`!nk zva;$3A&bi&Z?G{p=Q|ku5FzF{`a~3m44~`ww%3Gbughu7%eqbSzzyfHIq|N6dBhK?2S%M)g;bb)?ME960ZIE z+^_LP1GyjO2Q#QrsqPW;$};C-xY+srv;dDZJR!-7aE== zcs_r6k8&t+xxK9M-vwl@b*`tB83pe8xyWH)#$V7RiWcuL_O$ zF*Y-A6}yXEe8T?B9+%hfHDXs8nsx1D>)^l$lb>u?VKd@#Ii9grry>>eo|xeNK07(hzM>-pSJ|s0rv4hG)^Q8CTek*Bb*{*nDTCNQjoO1W?JSD zRA07)h4CIok&-142U7cwjE{R1xSw`#=V}nn?+QRxF};NAm6r$j>a!*baAg*-P9)Iw zj*hAoZcV+e?oyTZy9Q?nso+ya55AKenNV^QdVEk7bsx%jprGJ@8@TaJZN}u2g1kIp zjx-@Wyd^m{9a_8LQYHA0D_A9yw%Od9DqX{MYtDy>+!AOBX?PL^a5!JGeCE^3jtA?# zf)FF=#U}y@WhMN{C2lRLzNi8PMQfT$9=X^DPW3U`FbxQI$FOHOEpvf2zL$!p?H2K! z6cx12?kqd>8!WG^qXz`k%SVrjdgIU5GrP6>lZC0sYJFORR|Z*AR_ueb;vm`_6Vua2 z0kkz1y)}nqBo6j-Rr7nEci%sGV#SnD4I;(hSB7(+l1FWcb^_X$+`Mj%_O^Ib{LV(_ z~w6FEt*T9X(^IkGlZz;HzY5=xLhEMK}tRAey$a z+VAxh?$|cTqwQg1RF-?xc_m>zdZfFzRTAb=?|0atlHyyTubCVko&oi~f_^|=T!jyW zM`bHR>{y;XTkjgp*Au4^R1>{IMMbqaMDA?o`z$aD191<(IkVN(Rj{DcLq>D#{th7{0ati?yg#meDKi_vweILrr?12{ge(hfym>9rVa%w^$IoA1YsM&}#K_pr{ar?^ z`gCuhS;AqCZMO0zo5zy#^GkvnE z0>$Npg}tVO!Qtz(U~%`J^(qzd^V8v~8{3H%uAbiu&B;Ei*2kGakEz00@qg@Yb@X#O zy14l)1Zdix)o72^mF$RXYHDIOue((Ews#j#z_XORa&mJwGGmQ%a|*T)2p;GPPXfOK zz~^ro4y(^?*1UEbj#EN5#~d78TtbYh7q=!4lDr`cd)_|xPtP&KyWX662E=k0Ckc4O zjK$YvjsKM}`c2Izv?*yH5s20X*_ zeq!JBdx!Sta{}*T_{S1Y3##cVmF)?M=MhLz8tOHzF9T*{WkDihMVXn^T#SQW1C9?K zeA*#J=Rn;KavZ7Y-r!lg{(R%I7V3dx{cip4JIWV`i~GL|wh|fP&e8lQ0;L*g1ssXB zG_HF-Z?~8zQ-BaT3a08^-!2B0o{w zP|{WM?`&f1t>+gKUoQ-*UrmpMj}l$=jRU?mCMS?X#9(-Jq1?RF`IFQ zd?EUAiqO?V>-L=sNagJZM42tO5ePU@*ef~K0yg&3yOareY8eScFp{R?&U{#AKvI&@ zZn8Ic=R1yWT>=bqPD*N}VP474YMmDO_)-z}B19idu61q9tpW#lXKNX$z&lgBok3wU z%US|Wk_Xd-DcDrKZAVU@L>g9=ck)LEOsHSi>x54SQz^%2Q+gQ;6si@E@P{!F_$0GR%K_L!J;Xer4BPTHa5UbX+bwZ zZ+dBQv7_6DfZrUsgsi7$t1+X>_{a#xU$?PPb_|Eb<|d1P@_2=N?7m(FZg4sq8{3Ql zs8|N?A0vyPM;1<5gW)_gQ-d;7xgGE!&fn6D7Uet!J{Vg2R`SmYua3@nY!96x%c zLq1pCG&n&+anD4-X;EiXXRhQI9VZLA6xL*ArIdma9=6Cuk23^9eYQ4(;zMR*Yg^;Z zH#jrn{j!BiSXe*kWxVk#*M8q>yO~fcJvlVhA^G5RwC2m&5vV>=79vuwsn;WRynjsX zpOc(tV^j}ir=_K(qkASQnxv$|H)SGJn08&-CC?OWLd;_-JuJWTLQ^XUn_PqqU#~a$ z=yO88f{#x#IGIVM&>OdstcsnW*0Aa6X~P;<&C$?8=cO4rNKv8BULmFKBZzk6D{bwY zX{vTVA=fw8Gn4o3zx{!}*)QX}y}R*aF)q_T+&HZD#(R0ZI;>YC6Yd{g@>JE$t^A8Z zV^8~k?8 zh#JqI_BAy%6(8u9x*WjdR|cqGgoa}IrET~qqwsCt?!+c&(VF*k<3J%1A_eSH=Lb|=Qikdn4gmdgB= zWo>yX@#OL!TmZ)QG|3O}0ye?(9-Ps;jLj9#c`!5MN}rOel*fGi{PhyLJD0S(eTvMl zr=_8NjQ5x?&)Rx9MIDd$c!)KaD@o}Z;b$D#y`s+0zrMTi0&Qcw|19ED_7Bd{Q2`tY zHntvkKofHmm4;n<{>-x)JqQFM>~g$YS9dp8Q5sG6q)Wj+{ABJ%N7H1MmGg&+aKrNA zki&7oCyK3Y%>ZHX)0djVRw9)Blatm%E$A268&Ab7;DIGmZK2_Lgm}J0-ALXpWEg*2&S)WZj0_@y5G5a|v@3#U@LuRKCxu zywB7FR|?z!nG?s7F*`IN!}3k6wuZDcrWtW$bfPp99*LOZ5%PB$_3^R$8mB>LSKra` zKArzeM@PCfbn6=vqs#+QG7}FdEp4r$1~>E#z)j~;2d>C~^4TJ*iPS{+9$s-%dCpZtj1H1_dq?+XKBSH?bPefN@PBRx= z1Op}qGlPFsbxpAfSP4gs%|l2@Jhv)Ob8bF^DS%AkRwfXF7OUnDhWRX-edBE>8ktB6W}UZRg`J*R|WTmdT!a zKd+zh@d~e3$!TixQMZha!VD)1^76EbmL?HV?cdR;?d+T@64T$xXVM6+1ZICq2rH@_ z=*k^^ne9JzVwnm=7w^*?2wczP6ZiONp%kH@+Fvs;c zc?E~F2d9ur-b3u)ob2^dOkKs0VvyD5-spUM8kLJJn&Uf#qTP4CYp7cLls=US*injp?J*2n>@8w|PjFQCo}%UW>!aOW zrvnQ)YwO9M@5M+tb@lWVtNbqgFWO`0Sf(3qvh^!p&aAX?r^V`BlA;Hk!1}AA zFV)r6F|eS#ycSBz9slq=7H(>=8H481D`_dIOV7*YvfQZJ<1sT_Tobv@T+#~~|GL9P z76kGSh?-El z)cRXO{ugK|1(mSiv#-Iywl+3mE;|ws2$_D43hj=)J-9Cc0j=-vTyIU;WpB`@p@pRr zH(eWdYwFZZryK~;ap)-|dzsj!V{!j~8@(nnp-#bB67H!Rg5WU~q zHLx~8-^b^wwOiXl?CD1w=k+xppA~ZG=&^(wj5Ab}!ZRHW7K(Kn_Md)^(iU2#68lv@ zzI#>{F+sMJudGcLl=ziSj*m^dQSSmNSu5*^iK&U1jY>3$(XlZA4u+=ZT(pIlOjC*5 z_k@XGdGE{w=<-l8_$LZ`0O=4Xvv#W%&57S8Ce5FmzBZwW1Pk!;E)Qk0A|IV&A9r{3 z)VQ*pr6UL=@5*Gc5eS?*^7BgA>!^dw9g2&9@dxz_sjkpp92IopXRy6uz_FFlu z+C_eZl``LPX}2b-za*u!ep$W1v})9hLp(vChKai`EW&1uy_L|MJUsM&pPGHVe~v^_ zd!(5NvWArub74P1Z-BDD7@a?~j_U309U8JXsGjmILU$iAe1;9Ks^~~Gb5sMO((j^H zfAnD^cmh17r${iU^H>~AW!AsFlP$$Tp^SHFv{G`zKevV{%@9R2Xbx8Sn=(J1tTM)y`wpY(h`@3i(}T0 zyKbY8e+Z@{)u<`M+L!if%sWIV!?LX7v!-`RQr&OfiQVCLCEuA%&_0Y zO|L}q>c#7Q_HQdFHGg?M(sEEv@YjmT%xt~q?Tvc(#Yw1!8>4!B7ObqKtW3wseDXhp z);=*NFJJ0NNZ^@#{sgM$ak|-+FYBk7f%uz(F=QXh8%;oFT!@5&}*Wn%fsY z-z=oN-9Gu>mvP=Pqu6KT?c*K1r0Ax-5@ihe-45cSTdj(gRXW_#UjXDe$+ z5hoDV(o!}q;$-^1v{cjeaBpwvWP1ws1dq=w8C?A5M2Tc5NoK?WFc|yA#E_aSTD4~4 z_wuZ;{+$R~(c_&YT;lZ%Bg)5;Y;3ZMn4mkozNJ-)n1X_kf+8l#u+-c0vc}DFdEf1u z&EJkpT8(8RD@!O-ZDD?1O<#SGA*!NFuWXEs!*^%?SAlu=b@31G1OZORzC;uvqJ`}Z zhl_hZbilSFJ86=F?Ckqz2luck;zK;|gVB}?Ue~R*P4UMmrU3!5k)Un@3_*A_1irCh z4V0!v;`vljQI8&Ay-|vD=At(zp{C;qEihDAT9|I_$ROokM<1Q>niNqnwl%;%qGT; z(5RdC?Favr7|TD`OH=a~jI;-W)X_bGorJDLTqjpOxcyF0A&d(D#;Dh5CDFc5anVl= zbb3HSd~fak^EE5I&g<_?!C5d~xRo8u(Lqf|hlJ4;s!ZvQLez{-(b$NRVjoR5W;pNcCE7mbRbvm25a#oX>s$Ffk&<>DwSZujl zG1C?}G8*A|Agie8)%%!{@R_i_lnfP>xG$p65S>PqXO#pD<_apW1A>97I=Yi3VeISy z{^9zqf_jki(=AHDA+y~%lK1bGkt^KXElnR-+*KNblm}8Wv@bUd0wR8Cj|4!QHhUbVn!L zx#a5#ahxyoGNH?qLeaxRG%4cZzX#GLzgdDK19zkiVqahv)rlX>FCm*$Wsnv=EMGkq zpPbAO8G8xWeXRNFmEZjHv1sN8>S{5#V7vE@sd8XHU&+ReT)Lt^9;`jso)aDI@4v%? z3C8QtT~Ov5(UOZN!*@PAYV^w|h~>s?UfM&)b+%nxTPt>2`W8agUaYc-{9G@Jj!7qe zJz?~|*ZkJEg_lYYeLg%9iD#f`NyKppalRs(Y)G<2gjV1Bo#g^Y4@J_2u!~|)TYY}g zR9`q>sBY+Sd#nCc&H5is=g0^g zH9-|QfgwJz4B&X;Fy3|@B^ys_)2asve%V{=PXCqcq|@RyVK>YFX(J;kwO5zJrh6?YUjdWWz|25{BQCzn7v$>v;R#C zlo@d#*H;PZY?w2gPgK-ytRP6Qlp!xQ_4?=g0%mortPD0<|CZK>a)_J(If+1F{VH%h zYyH?$^cZA!Aqfk*NRFfBVv5Q7l-|Ly8DZ=cT8a~mQU`+FJws&`?b z=Hylx`5`idcw9Mr%LM@oG#-d}Hi?whbsEfW1OYDx1Wb3N3ot#|(wQ<(yr`~q=WI)<$J)u6`@6O-WSKs}bK z(P6ysuZSFy1K_O(y60z5sdbGU7xplN&E{`Om!^_1m>1s@t9 zD}i*g%c8Z+z5A2=a(U8K|Eu*6$--OMBwLGv(CCKdP!oO|`UcVb+&ehj-G}dw_Dno3 z@W=j<{t*4!y_4JK34vD~#XI=gcdcFG3sIu+k@}6rzwc0{rM#N;7MC7-y;n=gqkexTwU$d{>sfU6XI#b* zHwlvhjvx7x4ZwQz9pT2oRfC8^xb=D7XChxj$fv@j9#= ze~>%2|FcIoh7YTN67semY*fym8W^oT(N-p=|xThA)R7}Ulqwf7*C zrACK-g)udplwbUZDIF|~cDdH7^KR&Q(>9c-o+#q_;oiPqpQ3?`hX=kcm8gbWWWTlj z>$IA%iXr9dUZvT9EcR`f+iC}Kx;HEqY}MdwXx%5rDlc#q$&;PQk8$OGeFEM)BD!4G z*18+*9*(Tx2Bk(L^NQY6<`|QfC(rwr`km=tWo-`^%QCRdhKA!q$tm7xKW!M3vwI|X zxZc@=Yl`t7J;EkhbI(uQ@Y<<0!S>!kN-jkHF`B1f%Ye+pcQ%g{z0PrrQha#-5DCGAcc0|)WXC` zQl-wu?LuqAqq~~-!jr3UdtdAcnf3_E*~@db2+EssbY0__?kz*JiYtL+2|~vUs$ItX zyt2~-g?9wuwjV#;b|c}jFn$XSxx64UG0h?ihU7s}yYMS#)7iGnYK9FD_s=uf%KlQi z`nLmaE*k4^*0#(N?(H%az9O*o^ZXH7RI6+JO`g2MrBn<5G#ifC_K|)xa9tkyvF_rY z#Tn+TkZ6GM$}t4viMUya$VLhq07g9R-q6gn8YSFQ!4lD*2^D$em`Wx`@&)z*YGKVKs9BlZucDzxM80{Bg)>qXW?Y^1=Jgr#je=1IXx)w*O+9U1?zB&v#Es%l4{GUiplJjS}D; z(=;%w7Se|GcC&#^hOyo&J249>b!&+%zL_hHwG~#lsBlp`OrGU}09<{JUpyB&X{f28 zy}hOx0CJnJ1Uku&yVjLVjHNsp8X;!j<;%TYtg- zV0cSN2s}G^`%gDp=8pnPkuQ5Bu}1{ln&^Yik8P*+Y<(X|J&Tg7D`?@2B9dWHR{HCU zx-w>52J7rjQ2fzV zVD12|JP?20Y|34z7zFgJ**u>)kowSZO^;&wjJl1kUA}DdT*@?ifMTNEdviLUlC3~K z?&IVB2Re88`O`FHy}i{#b*k^Pv!{QYVkJ6%x^FROmy-GlLRwZ;!x9B~e%7U$xF5W1 zC>R>b8^(<1=P~A)lwzVHugCMs)YpHu?CoO{e&R$%F2B7w47`EV)tnOF{{`uGzt5^b zE<0S6^6Y!>y_j4P5e7=imK-(x&=pzP_44xLRqXIp;YytPTP>!qHbK?Z(I4KWr$@(# z^eft|g-%WNcXV**ta_$hhCJf{Y1EBVFO!0B9?7JaM@*2pWS5lo4~1P<%rI^UaDQEwqm>+Vpy{(P^>DbI-&wUaS6A*dY+gpZN zML9J%943nBG>U58+;kw+G`0kXE2@X04OQ%mS(%JGBJQZxH;5!9S&@Ke*!k4kc|}gu z9@1tw_GW&P6mq>8^CB`T*Rllj_7Wh)*}P|@T3@*w$15UO zV}z0erJZeXEcG_f763(9 zEBpt5yM6x&aN=E>!pP)c1a*O@kU2uXrvF1ykE5Ke4y2KOa+*tziqo zbKbl6O-7CAboS7SDfg_##XYcH?=tI@bF^EGOA*yJv_-J7uspVi^!1BpbeMZb#&Dn{ zZ)G)z9bVJWQ2$}n_kS^eat_kXe(&F74h_-t1F=kSI29fCurL)3+{c2r8DMH-^5*G zs)$fJ#S0fJZn5PxHMzR1btuZTf%M-OFMd{0X{H3G1f2_m7r%=O3J&`Rc+iUou`n6S z*%vo=amyG&q1^a&SLuyjHo zUoLlG>U($B##xx_IhgBZ4IkW(=uDQ%h@5@t2wxqFSXi4#*W~t?X~$x3Pm`PqneH!6 zLf0Y|7ZX-O^L1m(ZN{aU0h)pwM&*wmXI=-!gAua9(+F;Vpa(#lEm!dN$58nmlg2q* zB!lNbLg72MADz~KuR)}})X$8M29v=pUL^lLT<$G6!VEkjt zMXx8Xtv*+L<6QA|moR9*RBKQ-Lw_dtl%SKz+&UYR4Y|JxN8C3e90qeWp*+Uc<2yN) zvhv&j!89#8A2W4r?Noy5y&EwxR53Ag__gszC~2{}pV`@DrS4Ip#(j#>%K|%FTQ4_^ zEG7*uQhcZanWOxG4W_Y9EX5tf*J`xO+w=!{VP|(&E0uHf@|7g%Zk}8GDnEGf7CY5D zbB zSKHcniP3Ah6us@@Qj#kQGb4Dwjn2p~Ow(^)nu=(%mV;BVuQ)IYMvW#CYPv9nH^$G)nB69_~V{tYNA-6$xP@ z0ow|6bmi{q*wwi~PJ5?lVb{+iV>O(~LNZc*_hMAEwZVU;5p);@$?|9fw**pB!U}5+ zszO717qOWf@b$^UTYQ$fgcHS0D zX3%4~zynO$`5;x4xds(gOzn}1in8)VhU{0gtzI=s({bI+2F3@0kX6e_imBOXhjNns zJH}Nt`7xsMO(9uQGB1yDAvL$!gKc#ZwYj&6gnC6YP`e(j6cWrcH$!MVR#(x1p$w+k zAkeuv@3xRr5r3ysu726DWEqj-J1XkQ;c^}r3rk_19Nn$#7cw+m-_M7FL z;MShDMvm4P&{(E#ZOZ~_A|rjPuhNy;pC0nj+Q=G`(DgZ=xCOudY5|w;kgrkuNjcnJJ0=mDLEU0@H*;zJ8*h zYXLDx0|BH{(WMI)1k%$U_bVmguFn}$ z$!BCQy2#0W_=-pHATkA?ILRrZt%j)R68SvJDq~{ml}wNvEG*Zr-ZnN?=UZBeVxSlD z_s`tlvjV|(+1V@PrGX3#{FjjUsQjN4gFA}64?Fn1_{h3=t7K z3bkM}ZYBA0rdjp%Yc7A^0bd2s4Lwi9Lg!I*bQOpNZS;|y#q=o?7~py^s31Yw=!RW* zIy~#{-H`iSgM0fdLDEuwT%sp?NrG?XX^(_1N(0NXRs@GKX>$&A$CWV;IBD_Tramz?L+(nAkk_M2Xj zAf`QLuJVH0v4ML!5v(srS6a2DW)HU!AT-#{z)5Chm{1qse+X&MCkx}yKWc}pCbCmU zTf2O^8c%ZC<^x_twFiC;O}Kv2HVI>S1}>S~^BrYAlaXm#NyT6=ir0Z!1JQazJFmU) z(@?tpRCfvP)c%#{ZJ5va%p8=A$^}=V6h-%k=No}X>$S7$=AG=!3_?y?gx?hwsDu|j zr`1vga|6F1f2}14`klW7E__Hli)4Cpx~{SyfgtcLq!c?0=4|(62Ms^*o;TDzB?w|L z`}|JHwT#oA^ELR6PgBXw2t@&wtEUZUIe}lPz)ugS|pH$W}RL*!m^esR?E zHg0Gs<#TGloMEMYdMzpaUt(Lr3i1Qzg9u5rzBFuaLbtS1E=;^QtD-QzomW{ibiBOh zQ^QTBR!Ko}xSa?i9AS=xFs6?w5GB>lx?9q+ThNLQgG!h;{ zyVC3WJy6*sDpEBh{4wT-9B`nxaJpr(sHph-7eTzxQ1ru`b<%SI z)x4I6I0omGV>iTCPJ*<%MdK|^fnAQ;ZJ7P^>;ih7md`yYH*o}2|0&qd&urQ-IUW;N z+vAMWL(qHg{cfS*LEtq#cD#&@?4%b%>0oqFbua7yzd3X!63p=2Hf;f`p>MO}fl+KV z#}25^`T5q3!Vz05!Npv&=toC1)VstWt%dz~g1R6Z=I_CRu)|4TucbrzEwZ+`AQY^{<4blz&jo)2qVs<##iDyG&d(+`~e}RWJA>$q@u!*=mV{AfF zl=jMC-oY!AG~-{O0|$%gp-5O@r(k@knX{8;JshB-zre-AKua&`Iot4U5B~u^pVaYU zxP}i#5SO#k+chH`EKfyd{8*@^O7S`x%0ekV{SdFdFf4V~da|aApUn&k<9FX_T`CM% zJM#R+76e5=VYpt|2If1O{Wl5A9q6Wx&;rcDLF4b;Zh<$M)worGfN)cK_qAa}y2f)l zsMgF-u6?)YG7F%TR;zg@6X_rhGCYf?@kF9g{0NoXoGHHQY!k&QLu9!&s9^& zk~gh~K?^xg2I67C@pRU`n0V6?v*{w&F0R5<2_dINBLqUgb}ccWR&?aOnC?OVSc& zpf)Jf%5knPOTXZ|_1)9~RMga(wk~UW1;4tP23CG1=}V@#WoCJ&t7a*%N`=nbSZ|{G zF>&QRtRi%Z9dwGp3E>SHvM}F{xC|HQRux9Kmgwz^7cXcZo39*!Q@7v$8aO8V0DO9S zIi9$3X-ewp>svl>cYjsIhw1^!T5Fr&v2jO&eJ!1#-VunbmXZ|!H{mnaj)|$Tt=+G1 zL>zvofJFIxRqHaFzA9>4w_z;04$og4*_BCF$Z}XE481K|5`6$DSS6fLz~LvJ+P7rM z@lIM}@U1Nx_;>2*sqx`VPC|(ogk9{M)ted%ptGVIIa&02`fA0{W^(%MwV`4O6TXLb z%7Q4@Bi~f>qg6MwXcy|x`a|lTwOOXI*r#}XGa=qrb@}Tj=L`pFtTi=bZNnIH4Q%JJ zyK3lw#h>VmGP+~w@P)*nhNV7)IKuboZnQ+D%c_He!Y{!abbH1`kY|bQ+Lf-TgP~$M zi2@^)8xbtBv1p+f&|mMZ*&I*^kfV5veQeMghue@CvlvKO!foOdCTdLf>i)k z9bPenqMutCeT$o#f^i1vr8pkQxw*M@F7ZUjLZ4>vu9n`H{lEot+fo~73DSFHJ%V&z zd)${=mn3M+F=<%sq4@cEdXG6ToTr8){!#8h>Cekv$KxzN9@hwF4p@?sa#xocDJLTe zv%a`Lm9#y1P&&T&1@88AQjR{w3nxGrN^n}AsNP=SuEIPi=}aB^P5}hSgYKDHwDU>T zhTAr^L9MrbR-b+=%mj39sjhD#Ntm!T#|($J4BPsKG0bRIM^znd<4Z@V#|{Ry`swCZ zsx5aDem8hZz1({^nH`hs2{m{(vT@P9FvO$$%T8<&tk!LHWH&`5`II0G)CTS>&$QIK z!_!I7U#GU_(!dR~pPt79leH>mTvfHm$1BQ(19asVd~?OAg6P~qfE-%mc;tD=D;Zq_ z#!?56-)s^fur!Q847%YCvL8)>#V8vpP>~|?i|tD81ZsU%YP|}2EQViG1Ze1#J#9Zy zvYI+wiZ`Vn*;@{A)Ofu?bXY`Fx6<`*hweNoURfWnB;ek-@!ixNjOPJrlkWR6JmEoE zgFo`scVuPHO=k{H+$qhAlp=@wvew2CRW;q8hK}Zc8onwnDoRV69xERUY>FTvqU6kP zU=2m(E|zoZUxz@lvu>WO4u0w9?zoYN#s?epNtl5Ieshc>6~q_i;uw$EWLJu0W+vI) zZDeD?_INobXYkaq01}$Pz;ya~c4qx-bL_*Pzf6~Wb}HqyN(jUj X`n*5%RwE@&zab-~AXy-8^ya?+Th!kC diff --git a/docs/topics/_images/firebug2.png b/docs/topics/_images/firebug2.png deleted file mode 100644 index 4cab634311cdc6f7349c70d5f30613484a770009..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69392 zcmbTe1ymeS)-_59!9s9C2pZfWxJz(?ySoJq?vOx$;O_1Y!5ut$uG^oi8gQ4vI5tWO-wN_N3K#QKkdA0C0@aGEtCeJq^Cq986jxCy zsjI)Fq;#T6Gr^Ggah|s!EQ<{HmEaqM7_3-#T?=o_5~@HswXy8dP+MCCCnM)+^I?C( zsnJD7|0^~3>ob%r#y7$}3W+IS5%+bfaEn%GK17y732&#l*OwTj&2=I)Tj*q+e~o=; zRGb{cRCgHac8hI9;13HFDWt>IXAJCM5-5JBAB^c?w#L|V)lvmfyE6A(B4F^;$szxz zymF`)8H2X}^Bb3MXJ_Z&sJG1y2nY}t6MH5?vg61a|2)>b{NCbr4}5Ve%51Mcw_*<_J4#uJC$m!* z*MW4lZ1V+;i*lEQNFzpLxuTsN1->MM$LGl0q&WMQuSRLnr`X-vgeUn2k)EMdearjWHJ@bqp06qSs}x_df&E~nY!&4C1t8VW^;=2tk)VNYLlL+ zSqZnmzc|_X>Gg`2vm`;`8~Yr+_@;ZaqX9YN25N4zl&|aZ{azy)N_59HAr$|OTi#;_ zOG9XGA}VXZ)?z8X+h<(Hb9j+=B1SY72s2CS^WX7cDWvP#(`CY>y{M<~cZ-Y1>{N-V zlJV=(GuWg}J{8_}uy8(ZC>2+G-RHD>s;)9jm1T9cQI@6_Fw;+-Bpc&3wy+MQp=EeV zqE^plkg{(uC`6tsbPo*J+uBx?o0^uTr4cYQN4M&1@9doKP0iSxDXL53qmX`MV`dYvj;h7_e*yB6sA{((&0;|j^ z0@#z?{Ou!Xqse<{T+QAr=gmDL+-N`Xz8-1zvWR!rAP%}5clY)TXzxFzT9qZ14{Pj0 z6mE5gS`=RGafIHhi8BtLoV`atsK#P!Qz({8fTk}CMcd1O^rNY$oKTdy)=Nx2p)KP? zPW^r`g|vvT;c*GCOe^3Q#0-C(Uz3>2)1z?*C0g`Jw&dv~4!of)aGy>hJ-n$aVLdNc zL7@?YKMaItd%GYS0;u3e_7gm%4O+hsYUxSdW&e z%fR=hE{rMh37{Sg!6MExtw8z*}l?6I-Vj*5mN3jz1{c2_xQ3h zTM>s{`F5Y!mk_JHPPb1ylZdjAidfE{@X@;Mce8XUo^d95z3xHbUaqM^FIh^FD#uBk zO_e}+5bnqAPscdcnUHiB#8rz^9Aq>pMCM^HvDY0-ZQP1&8OC{2Rai*2g#IV@;~_C^kCK{N z+t9nqoBX$BqJw(k)%T_AT~>YHEkt$HxdQnScEUqJe&zeT*&p8(iod3cg@yJ0{d;o0*~1fj z)_$8ltD31OOjfC~fu;+XhOcC1%RMD`N=?qKR9@B@Nh&?ax9saF*ZBo8Bp9reHux+> zNaT}t%dGMN9*gY;?@4Upy_Bh%JdIRO=?^xnpm)+MLkbe9>c1*q-3$q*4WarM1{Jf zurM$%P@ZCG)S{KD2N_E(<#Wa-Q~j10Hf>Ik2~OVJwGDq`EC$?# zke=`Bf&yH>;N0RH6CFRJ%5E+;xgH-M(`&fCO~$brA3BAaJDhWjnU^+V!m@Pc zq9uZnSCL|?^LHUJmi2*#cxXr-F?;;0sR8uB^?eytLkP4WIXz7U@_v-T&pj`1wy#HM z`m@HEgFc`*JDQveW)XJo+$+v`Z4=w}!2Yn8?Wt>{-k6{ER!m@0Jf4h;ivw|k@}@C+ z&PzFBMJPsRBI|?s?ZtI{s#o6+dZrz_F&R~sn~unFmqxB*YJr?gPBttQbOO8;sAh@9 z0Zummi`7kL}gY$Awk@RR;|7gAnqXxVX5U)MD0 zYmQ7_SxqN=#xV(!jU8Udjl!?6UO~y95*}5CO*7x18L=u)cLh$8D>R0mr1MnN=6?p+ zX=Adf>kao1!CU~mUij*f@TSDn-AcV`8+p9Gq|L6%=*5iEOd}L(b!i97PRG`%q=W?1 zLc-d?K}fFoOu6Cx%{dAR3IzoPybt6@L0N_Cyxv}Mx4HiQpsuyn4cJw6r60V!w~gBi z(c0z}FA=Ig)l6|oZA!m#TkB+K8S3V_e%N-n59)YL{IbD$_YwV9QHhB=IVuu}@mW&M z8^qBTrS)^exa1JTjZ*AJ!_1e;A#`%DL(|fgqy;JUBwkfIMaW4r#0Eh~%Uw604n*6M z(Tz`|P%hq_Y<%QNZE?T70I`t5VJ}XOUOh{+9Br_@Wg#O^0)kwR)fy4YIEbKo`MvOT zaZ>5dY5_yf@NuWX;0jx@gIyvi(Ft(b^nG2P6s$V-#u3sd@?Tr z%1hA^gi7LINqxf}qq#%Ok^5M6tT&(Fu1_LeDO)%<$4y_sdNCcX8^z4_Qsl&NxRoSH zWWsvIsWPkch2~5!2oYKad~>QQkEgxvRg@)%?NL&gZpmpZQM(wMKuWq9J$qHP&R|z- zsnj*_kwCXW-8@ojc6OGMVmg>vSXh|S(VGH2prD|@Y^vn=`fjGem~`d>%dm_xwDUd;Zd%L+R?|unI z=(4_8x)MQ7})@U!FN!0Pb0uN^J;bC%ZBDT3p=Q96F)5# zV;42nUX^AfBO&EzZJGz#9Bf&nS>^cg$yc^qn)M?Rmu;JCi^!Ay@Klghl4h>)6!u!G|Iw$CZ*IVfJJeNdaLVfhH5^sP83>rz#GTGsq3qRuznKnA>y?LWwu@lidZ({risPo0lKe{rNm-~NO`jyQCE2h zt@x~u0eE+1vcLIb@Fe%1)7@7OGFa?xOL6r?#+!ml7W7ofK9viL2b7l5At--mQ`h~4 zV@t)kfT90#pxFKnazG64e1;>DC^2paP9vP}9V+=KTCE6+GL+B7uHCWH!H?VX+W<~X zHWdMohg!!1dvLawb1t_Xn#$#aP8iv1LwX9ML1F+odb*FaUf~H}L{&&Sxwse^8?QAO z52YBEslfYWG&l25Q1o8aIH zGq?S&!p2fjN=nLdVKQvlRME69{wWdFv0!i*9p=f#u`Kh8M;5=h)0XO)OG(w%7RY#X z!64mDhO?(XM-FLX8PT^G?4GP{H9rhiYFLtO2wiSMAux%K1qXKS zi!kQ)MK>et9=E?2V85?YA(@h%D7d31I8wsLPyO}{DqV&g9e!LeNOu0+s##@bA9$g+}j4STBMXFJ8E~xRjKX0K7q(?1rB^ zEB!Xi?Jzdxb-9y3=>pZ8rQg=Wqd8&b*0rTQlWT(`D2nQkJFs4E%h`|aQ58AQ7EtE7 z&XJb3ZSDH@z~MGnAUTE^XfzmUA2t>Cs%)U5TQGEhH@Y?(NI3RRO=Jck(a%|rZY)+M z(MJCx^u3{9iHnMYJJY(Dv$?T>frcipsA#C8L$CAm3&?wV9ngq4P5=uQ6j18x>swhJ z?u_N??|@Fh4-psDUl5#qUo2fs*{6if8FQ9oDeXq5_gAK5AAU23xaWyjHQ+D)ygQz= z;ZB%rXE&P5NQ$ImDbDOy{X`N;UhN@VN*%2-yE}L@Z?ktOkUY}p@HE^hO^@%`)oZmN$q73|CY7PzuK<0`%_N*`n=CB!*SfO z`_olQTpZrJRi)X@nQRgZ7kB^n@A(r-IEY3eWY3cNKpvyyQ^&UE3BxTLXuJ9vFXapJ z@0@Q*QQnjs>a<^1yK&=eqc4wTa+5_pBZ62iUXG)u{+bes7=#D$rGjO6;#h>W$a=Bq z7kgExViMW-XbuRNmLqj+2K?X9iyU3wLZBDrmJ2ulhI^wsmKj@TKO; z^w{IIH&L3RM8Dd_Ow!XtXJ1Z%~x{}^rXZ=}o^F<4@$+?7x zn2anoI@yW{ zkbnKJf!6zH!Hk8p8<}FANR}s^FmWsXs+ZU%&}!_S#|zNIN9`?EpSj%;Xw2P9&5#cx?oPtW$B)oo>9WT;vda@TXVh-!FL(*MyF#ve*Ui|B?ocu z_3!TPjv_EHF#P@f?Km{GwLNDmKM@HB{Tw|m{i(a;+4iHl+QHhoXJ`nY!2lM4AuVx- zxBZJEZL8)X&k`p?RJ@J9yf#^zK-2A%GZ`Ue8B)~cwagvJ;M{9+Ay}QUT=H0Dp zAoj@fcN?Jkp~Ihug>BpjG*#J?xAV;HCe0~568e)mBSYoui-r0rYVj(X=@e~n4skho zS|xmqvAG*&u;D`%q!mz~_^^)!;pmC*Hu$0qi5FS=4Au)4$_tcn+d7TAtY;~UMw`&Z z!kqr;qQZr$G<^s{48LNcg+&yUltr^pg&x*?XWUb@xCc9JURJzlo@57IdV6p{hKBAJ z*M}uB4eGr_3L#!Az4AEJI*Hc8=*5=7dn0k#LRHEnF6Waa&&Li&>0}NJY-}oISSZNW zct&G9UXP$`Wo6~V`8q~cR%PnfQxg*%9Ub}OXrSwkhzJiyK|}L~?DH!KAywl<>BW@@2b7-g|t(jQg zOpton@^E9u&(E)~Pn0^;Vz)Cok|_cvi6Nmt-&6ZM_j@X0$QM10yncGi`GsZ>iPm+o z%hr}%L2&-DKn1EGGds#o*t-J!NYjt8W(4EnikcaukfFm*&lT!h6V)%EdQIifHk z@el-gz+!_V9S=`Rzo6&NX!iQLL3AXvlS|_E`gi$_P=j`VPPk59iu14DFMsSyo@#5d z{rXjbH@{kZV)6E@&7QG?uue@3wp;=BN(oS z4{kOlHHl_6>XMcvJC$icD%4sn9^AFOY5jrJtp#E}(--@v_Pj4*b#+ZfcXHmhn4blb z<)0Jd+LA}4Com`X zy8M7OD1OFms|pmWEi6d$xdHjdTx27Ap9FPsa8Z^|uSrQsuV@3Bk43-|in^si3lO6k>@b}`M`>{?+xNA4ziO@U}-FlaY` zUy{fi6u~}8oEsukAj_bJocav3D60ntmZqk&yMn16zxYZS^uw4BJ~y7Z5}JonuUObva{@QbIx8abpFcxl7sQ+!2h5t^ZPm!Kg@L#!Zr=q45 z`#kDzzRGJSG^fXC_##dX?CPxZZ6bWh*j{LGY`IzuR2qWt@$q@}>ec6pO-@v6R2}mi z_p0u`c8vk^m367$#t{)h;g8x(Joh4>U3jTzQX2MxsH|g5?>jp+Xs1TYcXCI)Uc98$ zsQsbeg$VNEaTmV+&@%$qjNDvSvq|db&!0o_LqbCS`t?gKI;n3XLRIP6my(Dn^#E0}od7LSo`XQU_Z}lcy5!B4~AWb$WUlKt(lmb?@8flvgz6SC+ax z{X@m*Cpp{0?H@$c{p6Hs8Zgvr4kOf!>z+HOlE67X?xu0&$FeI2Hue`MR+dmk+IbM9 zA0UK9Mgo*MR;XMG#tWFZ)2*Rg@~+>@JNt=2^$pyhfeVSC<2CoPc7cpFiAa+Jq)KYV z=x!|M0Yz*baYaIMZ;yYg&T@mVuWw%r&0LG8r;}4{ette+J7O%Ytd^7{KYsj}oSdxt z?HM<6>58u>S(7!%_91b~)aSMX$F^$m(VQISx;oOo_t}eJW@g5CFiExAJU*ZT4DL-6 zZ<^_z%xb17wCDVtfZa2brM3sW&t;~D1^ZuhM@*^M&HM)-GG&8AJA17Qh^E+QBP z<1bNB=+VlbOgWw_C@DSMoesTw_ikF?pf|MMJNI(?`&g^w;mE#A)LFM`UE9IbA=QD# z@^>isaK2;WXnd}wH+^dFtB*Ek>P>S5--*;Qmsh~3f|j->A=@iA*Vw^qr{|qK0z`f4(BurxOuK~ivYush%1`d{IzsYF<;wCMG5>?&^F>o0EgX>-Kp<+*=crs=f2DvH_H>urgt5Kb0|WYK{>w zY-+LYTQF{{4P%?nxRJ_KXeS$h$17tMu&1{%IY*F@kx9$V-P+w1-$X#phdSA$n)o?3 zC+MiHs9jNdxu|GZWCIrv;qtOFG528w+dah6xyki~0IfiiFZSxe&Ea=OE5{BUq^fDoNdtOuA85toA))n7wXGnP)QRA_)s2=!Vg{ zSrTDKlOWUF^0kN-jQ_Xd3PKXr>-Rzfrj^N3v%3N%#yq}^5t#o1d-03lYS z8JLGN&=$Tn4n=%x^t6{r0sY6NTTk3URB53I7QRmPT2^0^89QzejvDv#tgI}FBsSUX zH?4xDuaC~EKg45K-kj_Z9ag}}$;rJz5Z>~*qvYV2oteSn@pu7o_#+X7A%w))^3EfF zyT2z=8rhgWZ#5ivVd-n)^W`{*q-PMg+xie+Y^fk1z{BC}{|(+l5K8?8{L96mY;Y-q zd&}Ogn##7ABlb5>(kTQ);}nYVQ0V$1e*G_*o}QkOk&y)=PXz@9(7v1zJsrT;4%a9m zN*r-^X6Nboc<^a(X^EGHMv^kxYvZYxSjOecy-n;F-x9IRBOgRlo81K^Xwb6`BSo<* z0uQOM(W7AUr%In=iu2LKB9r%K+#B_pHs!-3%#w3gFA6P= zGAXelma1Dk+;W?nr8Y-XFRouVfL6>u5nH&Yap*Dx!7xW-b$qJl$QPEFOy>>BDgF*J zB%C%k2JYg|egM=#lgBjV=AM;#Qbx11=SwzeyWUl;cxRv7GHSq$CQIZp91;P7rypUbXf0gNdv|v$Jpc zk$=4+3SRs`UE7);0WK!+_)jt<3NR9vj7p>cA0>W6>@sr968Ls)U!dMjlnx62S` zP89tz|5|E4xKId`r3`ZAH_IuHj;NS)n_FS7_X@!(#K!MQ<{p$!)@lqX{c6nh$Rkjy zi~!IK zwvXa-Iek7_=BJJA+ps~xUUAN?1>;y@$RumH&PSmT!pPfY11hK&p z&p&A@eYUYD|MUXN(R6CMJ0blos(G+r#M@8KjJ_F))OLg;)Ki#>c<;`FRIEO}n3*d}K>q z@m~=YZ>QYE?mgP#;*G9K{s9KnspW7MyOVYjcr{qf-jLW9iAmuGj$61f#;Q>sBI0~f zxyeA8^RR31=+t{zC4$15OKngX;VAg4tLHBNtCi^XWjslb=>pfeC%mqba#+)htPll| zbni+nRtXD_p9seHj04o>%f8>#)1rV?&*LVAKHBT`*o-h9)T_&5$VJR1s)M`#Z z5{-|KcXkSgi$*0RaOnJmpl&KC=p8BBp@kF?=>nutBQzr;BVH(s^1G+|vW%P7aJaTU z(Q8z{PL(V3@aWKbZ`Qk4V92X^|1+OXa!KYR(0%ue!t=*Qm6V-hjnYUyDd^1MR7;Rx z`qkFOh@glrqQKQLGxTX$oq6?^z?085-nR+Wc$@V-=jpC-%tpNG{@mEIb2YQ6re@H= zhim|-SN8jq;>A53qwLJE%vcY@!-n;oozE5)7H;YU$m5ffmWz#8{9#vDS07_(X;jOh zhF-zJ@gui)cf-y-y+4UPJUko(;0NUP`FR{^%)kzi_UCfs^7YEzxS>7`ieU9Od8g4V zz%mm3Rq96BTALa)Ck!=~sQCpFqsN5EvId!pI1 z(so%t%BxQ_X6*c5fLvitwHl?NQ8uG>$jrpjOkeeEHpvmSg*_lPCLEK>-pBS;Rhp}L zPK=EuGMmMG{rb#{;Muchy1Kf(K14)B*x1-(V`FQptCK~l-UOoN=0%~QLs!RZC{T5d zhg#<5zo)c4gGJ(*-b#|(2qJ&$+n_8h_>=vQE=CuuMBcKrvT=VcWcf%aY=52*`^u_) zyZ}WxF(WK=lE?)GgW7tDd zv{K8wwRBy-MPWLmw*@V(eqYwM57ETSmZ}kj{+_?U?c(C1ePYz4(jSd1COTTTfrNx4 zl1zGX>S;^ma<&l@gNv~=GgH^n3bBZdjh&mFU1g4niQ(Yn{5y9b1d}X`HV_-MTS{wf zzY?L63qp28DtH{7>2Yyz{Tp@Nah(z4$633>U9TAmdfyg=ik7R7=QGc#5!Fef)2>5b zq*rK%TZLq;VA-uwiVKaz(QTsCi8x)um;qWgL%Y_Foy5e%czAf6(12CPsp=^tAsQ#_E*0%rrSIEVZ*Ue)TrS~#HGExOu_D0l z`xshqA2e=#?6k{u)8y_2cnX(H^FXstmfdZjdC^rlx~!LkJfPA|*6^cw@>#~?f#>hr zfH-$Ww~~3#h@Kvr_ppk{orEk;`?;J|->$ss$8VvT%KRjmq+d<&3AxbcD$F55T4K z{Sqr53fDLRP5XEDg|BiF!0gc8Wp39emdF%;0NBjb6s8Nz)4K>(&7qr}9%%WE?w^cf zpxh8AJ1FAnMb(7sMoA=mo$Ssh<&>^_lVIY_|L67X5fri`7RVOhHv`)?bmdQ|KYI`$ zT>o?f>lj4TtsJ^*Hn#e8~LK+w2irseLW4-^+nS`E=`IVwuZ zgTQvxkO) z%F50@c4`#xLtLTCPuTSYjo60|kck5<@88EW81{CD6TL5MFot{m_rm!JCjraN&dv@Z zBI2~Ajg5`7vopX0&CL(dRLZDmXn;;F)p!7@`E+wYz1}WggSN$fZxS#KUS3|nx25Ps zDS1BM=vMb zk=4rMXt{MX`(s%<@BsXR?Te#lsC zB<%L|_a`wLhk)8yR(71kZaa<#=uid1#z9kVDk>^qN$Y;vOaNo-bX`i?o%MYnq#z)6-k$=kLa(c{}>rApcH`L*lH1H9^_R>gtR6I$H_Sv0uL+ z`H}HLYmA5R8WDUx$g&4WNU=KHNwPf(1&$+bQRTT>jEmKZ~GJ1FWX&axI z(5(IZOd8bA(vp%+N9GAegYGbJp3F>7n?{3!WHFbAhX*JvNccQ}DPV{XYSjTOcDcOT z<;6uvXecINu)p%v*@K`5oQ$5FW|Par++59@p8;&5NC#M!iuKlq7&}0+K1{1wNUYa_Ij0_WQRv zwL1{WU=VST13RX<0~2QqfR+5AQUo}BTvtonzbVTE<|z8(`O~!`M+Z2BjF=b!hYcCw zZ(o!sr1}4&(z;)2yL#qjQ}@}=GI&8VYa%uIxr2Dl$53|fpB4JO6lD? zJT|sBRjSR$_a#4{!i!)imAA$9((==%e4sZ24DfrVA~-1Mhrovq9}-s2yG0w-%*@O% zF=K%i_D;a}CrD#_oZGX^g9vse&>Uyl9m7;US|!g-u(zVm+ybExX}<>+l@!J_wQ+dV z{17A+@@$#Al+C=H_(0bp^jdZ$`cODDstMhbJ-xeYYF#bd*rn+*CsV3s$ac|z&SEbv z3(Mr*-rMgmhwSqw-@~1I#t31b5ZVn*`&U(5t``#5jGrzK7QlViaB{i|!Q})791!UK z*}2)-l1XgV;5GGX^O32kP=EhcB+#@fgfLq-HZ`U4xWiYFLO}c}b~4B>v6C_8ev7C0 z9~r_$E5e(v0plQKsTYciDx>DfTBfOjjh`$->0M{aP6Y1GU1g?F#Bp;fS(pakedyBZ2uLwV~9KQ74$N6D;aU)<{APcz$zPaYfx^o&UNNFbj5to~?!>o2CG9KZ+?y)S* z@M}Ml$C>xTC7U-{Kg(QG`Odtfp7g}2gsypv2!j~`a3pL=Vn9_@&3*&^9?e`$O9mLJ zIDL;5^0^@%t0h)L=|_~tj~s|^Ezrv%!Y@S$UQ;NPGckW7=pOTgDpSS7+8!1=?JvbO zUfWR)odOIy{rjm-d5@**`t^8n+Bu@|hj)P#pZwtW55mdv?jRBNh}#LrX3J_S3J!NQjGUUDws(Oii>w}^uQqiyq6MR;Y= zj+8Ty^p1B8+F!jAVkn>@o@TkPGpUzPkthH)j8w7zb5Ehc_vos)NzPDNa=A~Hl9ZJV zB5rmJ?_1SYjZ^!`P1~!ndPE3g2wQ#0pRd5Kp-m3=~gTNRU>zvW^vfr4x%q`ZAiDgWyX-fB8 z3kF^2jZd&K(hn9o@@wXqFEK86KKFg}Gv9By5jG1=%7(|~`uRcVH5_DD9G0R0qz`wY0 zM0E_XpzWE+l*Tz#k?<#duV6|f+I~fr?QyGj!#4Ii42w>lcETSU<1i(JMmbS<+YpsC zi*Pz~t}&Y^6|4`iYd^(Qg=7fS-IqznV0YCaZ%UKqD&Q+V@Xocna{E? z)O^l7coEzKtpDzI4AS>9Vm8!GWoatoqBg-}!0>-@&B3FbBa`-uE1uMSPN8;&?X95c z(p$1)Sn~$14bk{mjIJHS^`3p$CPH`O+O$_a*)tV-G!Mc#H{*QEqMKrGRFVFfF-Z^2iMG zfaSnv#V`Zfsv<=O^xFO#43%u!l^bKIUbAWztHU=Z9^U7rubpx07mY``Y`G~n7RIbM zt}@mGlQ?1&-ifGgL=YuXH44807w6Y^3Q{kZj*iTU8{yge##d^zKklzAoSEf+ZS^7* z(OCUSYtfwh-~!m6J4w1HOArdDa-%F;A;mQgukwjHYgQEjqp|$w>nLfO?-nIW^pXn& zjiR08=ER3@<$0>uDpPuj!>!E>5-h#Um*O<)XD;w}!brr25KtVyX{U9h+i&1hgHLb( zHlzLePkojcfijfA86smYl9*s{$bsWIzYGLT=Ss~`^%H+Gj~6WE^fnBNL>-@<^$!kq zIu509_vw|P!wb2)y8}YTqs2%~O-)a470_z?+uNd}UjQouTC5)u&ms73w`U7?ue_6P zeAT>pS#DN7jLEkxJ+3sGu@g}?j&|}p_K?nv_ema>VjWu5>kd+FQkB{As$HonM-)>p zn1iZlI*~E|uasjudZat%!gp-b zo($wfZfypS=Il-WZ_%8;5v_DQIJVgXGN`I&sswx-F26@|o~xKHY1b$LNzfdfavlbY zdS?1%-wo+6kEPv{FFevHP{+wu59gMys|DlmIr-|=ULRavymF;AmkD!|Un}^03qC)G za|8R0*$e1Pmn&oUt}cVy&Qb8fce*|!wymcZd}euSdx*x0=Q5K4cu*RoDYG7a1;1~< z1v%1?H>(gABudHiJ-z3|pp)h6Qx=qrc1#W>XEYg3`}XY{=x%;TC;bncj}kQx4<3J5 zlk4Rw%)wkusCYJ^_~BYR%MVtPvKe!ZWIze&#W6nt-KaYLLb_BQNMDFAk4dAL!Dc)Z zQq;$g@({T1NJxBIb*QzQ8`1WHLP8uI94^k!Uz~zsR6!_pwG3!$KFpCeUZ@wqD;H5x zUw}v^2tfeWMof$}P>}=&lvVUs-yN9iWW8W{0`>%}P!MU1a0u1l-!smqKH!l4N5Vf? zl>d#jIdpHIv7l4%Vr3046Xq<60#0XK+y!nu5g{SK`0-u-=^6x+kdK=F7r8BMYs?2Q zw!%H_OTDT|bZXq^8;_HhH%m8K+((MZrryhVx8kyfZKz}pjcO9Uw0-bCUIgC(+Y%V~(Zk~*?47{R;HT4D zz7J-zKEZ?gVBDhS_@ictFfirQ3$D>!r};@9NEpn14bzR9?q9z)E!+imD~yMLu{1Bq z>qV54n^@AIQAK7(hS5+;liRh;SH2_Ack7{SWr~CXGU<;EzZ8|jZ7Fz=wz>CZFa0%Czq~@)oJNk4shb-6Ph9Eo+N0ZBz43+C3!<}fAUt%5 zOjH(C{0D}dRO~xS_)k1-El>=2?*By8Y?_e}#io@PU)BPtSCX2grv@mSU44k6)Am_v zNoSa_BR_VDo)R738R{|oh-ZBCZN{iUIR=GwoRaK9 zfqMwO)5w{)0$T383e;2nq!O_&O3Wwm290}_eUh6$;cWZqa13u}w)Ssxe(&&lIZSIr znnMW-Ja|U4KF2L+Y%f&8lsvk2Lqm&nK#Kk^u}D|`i@KuX#`gAi3*aJ9QdC^=i;Isx zcH;WS@Lv8|Quh`tspH{@&sS!n87q4Q#I*9*;AxgjnQytCt8tDiuz`xSq+UIX`>pzp_ZBsBwy~F<`M}PV6h}xLM1OobeU?w(GE64jA1W7}6V`D84D#-Y zVuF8~nA%Te1(JRc5HMs=?K@ju)~VW19GxoBbOlS`zVZjAAiN<{4T(p%O);Ifu`gt2?)) zwL(7156u;$z(wkwwKg4}LkY9VZGPn}FQi*7Lo7o8`yrjlI9`S6lOW@8M4!y6uUdBf z2acLYMmJ0=QTck(XKE9$0@G==u0FOzITwa0CMS@mSRbjG;c_l|-s#Q22Iat2)tmg9 zep>DU3Qe_D9$4o2cX0&-f7q8VU%(nG>f|TxN$+j2FTr;e>KwVmhS0tZ2LnZnr!jo7 z%rqk!RP8I0D7Kq~K|kc$V6cpd7ca>!mU^MNI_(!gAo@bnD;Mdi`$$I*A6*DD+@+|S$@ZZ&ai|72xV%CZ0$+sVXg;hD*Qy%9pxyjd>#X%CP$KE08(tl*w?R}_Ql{ZV9(7Y+ ziQ#h*x48LldyJCNc%Cds?7VkJ&#Qx*7=RASk|>!w1_*Hko)4$mSbRIe=FtE%DE%5Dwy@+(2-TJ^ahw z?e2Gbdke@Hv23|*f~RA)3G0ujVx63Hb+R^?=jqRk$!eZe9w1pqEg{U_Tfr%!d?6?^ zhW-u+Bg|A6xrax_L-GLXBR65s#c6R0jSx`e|NJfh&vk?7-u%1t_nev1Z(I2?;tw?o^9zCEC7whJi%bKoM(3NSx< zE|@$ZxC85((e&D62Vn16J}>jRvC`0301mXl@o=%+FtEAVLqY-}=5Ij->@dJa2Vmhh zN0T$3i>oUjZPBIvaFIt!!bxwyN;7_J^PLxj=F0CxLOqn<&yYG+@?XSAfBfdg9zz?M z0W8QRQAvxvg^T5kFDyyb(RhJkB;*G>Y7}leASnY@HP*YVAeYT(?Og95N?&KWLVThzG;__ zfN6ly{Dg{kNY&kxN(~xoX8Yaoug_t?61DimL>Or3RlnNW+NkL0)~8>J0QBnW_lmD%y!ZNw zORdn;2jG_uh8TSSC(~2(Rl&VDl~qw^r{&O1^F#tYc0XM7F{~qM*^l>L1UYfzxKxk4 zwd{YW#V3zbpUdo(Fv30VADC~e&SpQBnhB*S8b;{PX|1Wof4bX-xj(dqmX5QicFj&8 zZ#RxOE*YZ*Ke|BH{~oBP`^Kp?hl%C6kFXq5Cem`dkD`aJp^Vd3;T;Lm0r>Ol_c61q z!JN~M&Cc!O!i8$&*u_M@q?63FlxYTBr@yHB{$X`+h9<^wf%ob2bh+S--;O5zWQrcW=jUV2?1s{C zhsq7YR^uopp?O2KI76K>Z8dkXBVMM=czMJwJX~|(j*6imWL>E}qDQT`#k6m#J8fnj z%A65;0H8^ZAsf{wv_eCrj0#PS{xUhEVQNI=k0^nx=snAmOo8d;fggp^e9A{S2)P0> zls$|~lNEz9ws-S|ltU|`H^#?tJrL^*=@MU<<(JMjlg{|lz`c4E-D5yA7B69>qM~AN z&s^Q$us>_NH3;`4Zy;l1yY7xtG-gSVz68wBpUkxV)~=o@3Cza~ayeN-uwV)h4V8`~ z0?1i^i*+NygMzqZ_IPXXY&oi#tEQX!SlPcb61_VKc~3OQj_w?yV9aEnR{$$!Vi1P9A0aKC|zzwEqB%qxE#L?Qeo&9rk_|e@!OU{cXfM)Sdn`e;8Pld8i zivz>XUmu^+ivWW}?`0y(^okfI?~a{?3*?k*(8yhBaaTXnSf<>pF^&bUc&5zPj;O|2 z7T$nyM4H)mjd;-~J8u*B13$JbT9nGE$UCacQo;$x=IwXIxGzjfwOiZR=+VFFMFCD% z2e1|wte{;1Xp(GpZqBSA{Vz`Vo6&}a?8a=nSenkEJitPr>WI>Cc^jMi5ig>o{nTcBIj@})M2o!(NKqZz>FwI{4 zfAb67dp0KS&MmDs!0m&mdC2%F#krwP68!83;wbMaTA^%SUP;)&b-{piNu@yG1zd|= z#Xj%oyN^*RScC_NEqs9jd%f)Ds+v{5`MxKVfP0s~QKrbn&0SGZalBAjSqYe!(v}A` z_IW|%_Wv~SN^?<>k%>5CKHa!4H4Lj@$@~R;VPTiqkE16eav>>3FiZDh{dXP@hp|X< zD#z(Mmth{W zJq7yA+0?M~67>5g0ZDyWKTIr2S!e=HS8sWf##ZarBy3|sJ&s2N*Q)Ur5otJNoxBcI zFf6UiOBmbOJy-mOd_$BoV`rNu$Nh^LOojvSmvG#mM)L7Z|7b!@%!z zfJQ(_Q1JJBojmpHp+r_Bsv)C_9lQFI+vukuLTqc01GA$xk2@5?v&IQvl%;2|xpkoYdGPWJ(w7?F~4 zaCj)K(f>~`><3;*h@%prq*q+mU&Bt~ORn4OD@aw=i;$eoiKC@un*CaSRTS^ql_VU- zy=Rua$*^9VS4N|;6_Km(;&B0t>R4p9MQXSrWXQoGMHPy4S6kY}1O)fu|3leVhE>&d zYg-5c(gKnaQc@z_(hbs$bayvMx3Hy4q`SMjTe`cu`I*czEI9 z-1YaSYxz*Gc{snUgKn5Kr-j95RUKQ9N8899Jqekzgp2-P%U@0WY;yw~(3?jROQ0nF z92%j`{$Q^5vA}tFc!s$LHs6*Qex)@F3aZ3lcBG1)ARK86GqW@$Vza z5b~&!bs|5_Q{MDR$|){O5kH^#-is#4_z46W0$cj7m4}f()qOFlT=JZJt>-P~`hfNZ z&^`G!O)vMW4{(NnK*fq!>BN9btW&iFp?sMnZm|&FJ^z*>WVe`Ps#f7gAiouz5uFA@ zLE@EF@RxBRqxDKR{&LD}5zi#^zD9Ga4QEd5f_RozmV;^hT?A!^onH}?ym^dD4rCFQ zTNB%0dWcTxz=wM8gOVLeNSZkCWwv#bit^ikV7J>dOu^Fp95*n!u_uf0e8`sN`I$OB zK4Yq1{ycB&DKER~*!Eb??`8w=p{`T$lo$2hC zkfiE$^({AjSMMJ4+An&^a*>LwtZ+a6*7|>D5xmgdB6^a6l#gi?2^(5R*l7c( zaNVIRA)(u-{&-T{0z@=r;kS{JVV>^E5`=f z47QDzmvZC_<=cJn%T0os*ToP6Ey=|59We1oy@e&~zo&R#|7agIvw8)mxDRQbdx zr*aC$Q-{n`2Y}=e%W!&4%KZ3p0O_8d-ekI1F~@Fz`ufQ>yj7z(J}KsWrtUZ9j4Oqv ziN=`3(KKE6b9F+;9$-Q!(h#S=;Z1K2&k_ubkQex+-JYf5VJ+oY^c}%D0|hng25kX+t?z)ZTAgMt`Gzbu%DB zB(rxG9;&S$XV=yOR6GD76?k`DFu&S@OUhZh%Oc@4AiGe}^2NA}@r4_$eG_ zs?eX=s*Dj3fqR8^uG?DuYSOo6MMbr+xes7G9UUDY&;VgGgYo#+09BwUVMGQtbWA@- zUzvT|A?j>9a1O(ln%g+Z0@>@@ zXUY>kR#<%%q}=_~&Af)PG0MP5RZ6c^Weo%-=waU|dbH%4w#qOYg^Z8i@A-%BVRxHi z(e7avv~h>o2ZsL9gW!D+T&y<$H6<{AWMO$L`B(bE*ANj{HE-&=Fzf%Vo}r}#%2xOP zE?>yiyIbxnjDzC$L+5;KC({>G#9KX$xHR!f8*}m>=*pfYsgxyNis2>b?l9ku(WGEo zP@nSa2y!#JO8R8}^mr%BrA*qQ(rbd)Sg{cw>46J? zjra;E+Hp}x>7b;g|S~DBxbpnPdxQ0DvXi~K`rsZ=5ce$xM5#9`IRFRHl-eXE7F!k zurvDoyVcw?UBhGrqg*NyBD)ouId`#s za_Kqv^Vj-KEm0*U<@Q13Ramoq4JVN!PRP}+;z4`&#kI4pnR}+g7}Fk> z;gUaQbEDhVdPq>MQ4XmlNSzVQMuxBBjc+HN zv&x{#jIIhj&vn!+3x3d-9JSMJY>(vv`qBOU{iUXnQI2bl6qzcZ?DA{^SR(+B21#eH zo?L;A4^6}6ugXd$9-+bsd{&#sciHjGn>?PT3WN|2ybqkNvHgJj76}RI&6`M&)T?*4 z$5j4vIc46RSNp4VMd#lJ2khVj7boOZ9yl{+%5%ze*#}4n(r5b9-rlEXTr+kL%5kB9 zkVo_3Q_uPQ($&cZz#bhPR$Fg?G#eT!YO7wU`^{b@P2(+!Jz3+;J{xE9Lo+n~JMWQ^ z5o)!X-lQ>cA9&9cVw>4T-GX5?h?o5_o;FNe zvs8<1feH`L;q0yDIxrqiiEqMa20l@{5ZM2uKQYwoP)R+Ik5<_oY)RkluF;~w%Vw2D z1L!<{ewI#^{P6|1o4qOhsYwtFm}=mq+_kcM9jw!DvR2Nu6Y?_NdOPx@W)5R{V9W-aeL(+g_lC14 zvsaWt1ccS%m-iAbN-0|!2a!lupWJfUVsPClAF_?g*A@_seE}C>pgjGl07FF$+v%umJY7zBEXA{RiMj17rX2w00w^7G z8-UZ_pCmu9yWhK5S<&H*f*KaS+vVrNiJh}+aL{Z(-{SEnW770I?&uYNHlqG~&5+{` z&qsK2YpjN;JaszqgV9eH7lS$t7rE$xQ}Vrc^<@<0?Dy4-qXyOx-?eK%Z8d^Jh(ZbWYV3DUwl0R2isT zesx55mEeM^WL`j<|LlQ8G4N@+v4poR8ZE}8pMv8zi_-6n2DT^yA_XS%1H>X*>RTUP;{e%HJO* zU3zQLG9xonGKSWZzEO${hfb#p0Ob#PX=n<;zspMi!;WGW_%9NRt(@QnB7A1ZnN0f1iB8*QXJ0`1W zL_w*7o+Th|iz_+kPu_svsFs#pWdU^uG!uKk2 zc2R|{YD$IWSI{Tdq2EoJ(Zt<*W*G>u*R$1kEpQ3wO@6ryY$Fk7tGIWkd1L#s&J+b4 zGjZC0o5RlK8NcUcAt6+u)XnTuxV<3me*Ocxgq10hT#xeAbZN?NNQynMEaAm1O1)l3 zmlyVeW=e{(teiBoKbTCzmrt9q{7^Hgq{Vg|MK$zKc5@TdYVGcp{O~D0d76j@#xE!FZk}a#m!$0CJN>NwD8&U=K(VQ2ZT#cZ?%O1 z_7Lc;0#8Br4;#3v8IsI+a5#^fuPTy*n8I|Ax&xE)ZRWK(N3=5~O^P_91;Fu2XD2Wi z%ftOH2bp~kduz7(Hdpr{h`4f8Ruue7GzsHM5P>kn)51$lLv-ex~eY+>rwHj!}McJ{ik{S4LeIWmHKm;tvV8Pem6}*|lp`x668v+v6c3^1AjH zW_9QxluLgDfwnNy`5N0DfCL6ITI_g*K)Q6LBLpHu+_m5N$?_0?kPl|->>3a==IR_F zohz>~X}^Nn*6G@d7cW5mMps`SluK}5vi!LS-LR9nZb!{MsN}>wOtj|0{EB;5iArO+ zY-l`v2xMeWZv_6~i;OF<$6@?L9Cpvxtih>(FGunN5e<;l5wfuVqzTbu5JI0=ud{BVEZu;XEX|ku*#TXD!3qDC{)-{3Tgj9mQD)=EYdCEc$96+*-1h`;zT2!KE!G-@@gB?$n>-qON@x%4;@LmeCg1%pE4ee&uz zrSk?wEX1xgb!oraZtv#24-VR*-^AUYslUQ|n1X)q$OM1$i4C{cT|`aCig}02xYtZ= z_@H;wMqrNrotC+KCg$Bi(#?V*B0H_M(jf+Z)bzc=68I;h5AlbC^=|Lh{}~Q!jVT>j zHSc`zavy!~cBEo*-tH79L?N`MFYT}YtoA*xk+hje&a#-}#OQVbuoqz%Ie0Y=4wGHX z3IjWK8vbkwqo#+<$z!DeTF;s&K>`UUSzBG5WJ?7Z`A1_nk*5SnN;13A;MJPsm9PVl z_FgMcHU^v!P4CCYhCBD`ZJDFu$ed07buI?J12vP-mNRGVY^=DZNhbQmRz1BR!v~ zuZ6@TXC$gfk3WcxMU^GL(CDY4WP}icF7-xH-2@Nc-iukMxT~~Z8NCcRd}PQPuo|)V zl0H&m4Pm*3rHLHUC!A8Nu@e&NDl;4%&sPEIPiF5Sz;#2wW3SR`hfR^11hONyXYa19=&@dp z75vRaQ;CVz1+-|TWHB%OGF|doQ*sDS{-%;(4twy98|KZ%+W-u$TQA3U#}5Bco)If> zsB!-m1G6W!&}0-rwMIWg+S3wWX9B{)@Q z#YAUEQ6n7XYeYm+4xfHKRHo7Tx;njqgaAJpK*>Qx zT`5Tb8D~Hv`^STeWzey~tk31V(U%fuR6eEG{+nVn^nLp)*ZRY>|BgSFk3b|hbD5D- z*OfM^$V4N>g2w#6L>AG1B8hW^{XX4h4fixRTSwY3$>mtr+X{+`c(Of-mQF4^OxDi> zL(VF35y|$Y%~~C_@W?7oY2moFferI%W{=R*s@$OtD zL_nJ{9+Yb&6L)x(b^eJ3U}XTnv}-FQp1*ku+Lp(V_|dvMCluD>yBO$SO;1FgDaX6h z79PM%`SK1sjQm&R^8H3)J8;o7C@90G-bZuN_yCFx4Wu)Ml3weY!`I7v(FzTIGob9o z)%rJ?u(e~-Hm8FC?b8X9warPzG^$(jy}A%(f`&W%4#BK zq)BT2)c%_riz+Y8<7573H0Bp@t93QMY+crYcfr?=LL1{q*f5r;*z8T80nPKQyu3E9 zZpYKjFWy7I+4{p{GnxQVGqoCxN?T?;VFsw? zo}Qi#>~bvs5#v0zVR&Njx~M}k-rnILlrdwhrT9WXB0Nm&$mgXQM_Oj)hsYkKQoV9+ zHIP&Y4GzX(G{KDdL&KUb>3qc{e)Zzm;g0z)iIz&hdQQ$=c0f>aJaUoj{@QE!sJ{NE z?Vqj46Y%k?EDux0#}hvB7=MZVPmGC2=ik8S$-fSsEdG;a`rijn{*?y#?}Puyhx{`` z@~`jw?}PupvoxVXMU#Y-M?k&&6W2X?g7s?m|Gvs5qI2frH<%UAo?AaUkT44p*cRhIL73Yzowj5l=*W)Ea`hN2crsvS>GT6CA5H{cu!F5&aJ zJow)?HXhfCx%zQ{rMXF^;gI5Tq3-EaX+OY{!4eA2K&s;*ZTj=2Z+y zSvR?#`VGl$uhGJ!2|Meu1;Huf10|1vP%XjblJ&>ICC}nw?W3lqE>FIS44?9dcI9r^ zc^sF9#oLjmqW0{S&#?e7>}QB(u~LP`bSRQj{l?Duk@>01bAqzHWyYZXZ#2;2RVDPD zMr->@PnSI!^&ZZg1}7AfG`1>-2i;R2cJcKq*Q1QC_+!f>-YZj0pDu`_vPxbax6^1A zvF{U2aD-Wl^~F%|$|7l5R;Hb5os|GbhfirnBRX{m8;pbbaA~ zZ04seiS?wig+-mx$%AwE%6H{Ww@FH|IjXE*$>ZDYV%t=$qD9Y%RDZk*mS3q5ok8Mx zPWW=NQ)aAd=)OBKBBVDY>VoRTY0z3C{UeR~eck*CEC$-N{N!O5lq<&MPS;RZ8h`m- zcyHGV4kpX@-t}XaTg~BCx4x?*#ew2`xXddW}VIOcB(C>qV$>Rac59z>Io#iGW)S#>hvx+P~hJn!viOu-zqxx1*W^6kIMnTFsJ@!Qq1(*;Ow&=n85MfYUw#$7!_z%dGolh%N$etgFf{+12wLPa zBDE4g!CZf|E(dQhXQXO$z7rnW>zPTnO@~Qgsb7ERE&ouVY(8#YU-gE7zVd>XbM4n? z5XD)foNrFMK01*%`l-sIG(K1OLI!d?5)ZUF4l&`xeZ#q4d;d$()}%ZRn3N4RPD$0% zci!kp%8Dt4Swjkx>}OK7BmSgsghJ%C!z;Rcr`%-j&u{NH{toiT3bkpy_PkQ(K0N|- zxMIDF7}NH=8EP}@0bvG6avw*4a{^(3fvn$`H`jr~xB@|rS}rOtB;T*ySszTqZiBUg zOn^oerdJYIuRX*Ys`IS`FnsuwWARiN0mn%cRGn@n@L_i4|J%iNZ{bc9dcLyfBF$ znEf{3AwC4&{_z+v1h<9;-&&@5Zn(YeUx?RVW|eBm%jFx*;qS;NX$Y&<=Dn@SP?2JK zrSVPU`ZC~dH&@2!$EBucMpFYf*3mrM&3ujK&VABd<(pEyD7}W8-r6n4t)iD;sg}V5 zgH6N~CKi}XPwzwqSIl~8$859my+nR10b)eLrC>2&z;GV6$(7SeMw2wr#)}FR4{xhu zHm(Iv&5P(0dbtGU816X(r}RiFMpX+)s0pY>JxgjzER-CI=&OWOY}`MYzwf<2xA&)i zK_IKH5sD-|k&VogG9vZw(VS`!xNU3C#F#PHlpfn0xyKK=^!HH|9f_XJs}fEp4{@{v zqsNDNp@TDvz0Nv~F}5y9O%!HrR#?qD^ZIqTZ$cl3VG$-059xjq=BOq7d_tQsFGYS% zqGat)@z2{%42EBj=aH}CTsF`NeYQXEdZ8k|6#WK4AWrPzXxFtq>pkUjmma5>>7DWG zZm}>0E^b#c4cUAKASN`fZpqtayi#8l;#at79CNmlK;=Eg2UHzVT$NzRPJvT6BF9}& zvGO${nTS7YBzMQg6h5=%@4P5z zgVZH_C2)RvKSp5p{@kyK3uR5RjMwr^w|AkpF zr0eA#jDH(Fy3ge)PWJ&e)=Q>#QnV5>iTx7 z^PaU#ZiHMd^@dhNl_B9fPGP0fQi*a=V;ohr?Rr;R1CkD_gZA!>O5!dE4>{LHkty45 zmeCb&DSfN!*K;tku_<4J`@&J3(`MoW3S08*xeg7U)fuwrb(|ab3Hw`{xtB$ zIB4Rbc%IX6V=WN}U0<`&M>nYF*yxy1eKO|v$L>HSwZO^`+7rH-*w?;i?MUYuW9~W9jrg%%XG?wgHyjf+m!|9JNB zyxNP-mdAltaid#@yz(O%n&|Q_z0%iUK5QxepP9=sw0~tbx&EOvZLo%{2D8sMcqG^oy7LPM)*n2%sX2q8Ttw@ySzpTw2?v#iTm-=9h;wYm3eKd@+#{=PRP^fm17CICdc|891iB6m+_)-M0~B`&qO5|Q;`^_^1`SJ@pU50yZR20s$-`w7 zmWsIGIV3nGmM_}1?!mj7ReYAY`rc|(s`VbKBZ1OYB17G_vbkCC^k~D#I8VNIJ5N5Z zV4MO{>u~v*Oz(!m0hACjq-Qk?2TJqVN~ctnmA7$T#=W2=e>^ds{>=GFwth?Y?Z?gqJa;aY9C&72826f-k!1pHJ-xXWNCzR@;RK34fOfTcN5FIN-#@=~3O5-E&2$5-h zXt^3&(+jw`!<9ysV30-qK+<|y1;eJm*2??5=oEfU1LogFeIT1 z4^b}!N=nk%KYq7h`i=pb!NdfSw+`Lgq}N=-UuoZYTR6_oIC-1+r;yGdVYn&jb-^5dJ0DTlIUhkCM;Hn0RPW#{dzk# zHGdRpJe^~`S2it#65gc2@W99c5u5Sx>&r{#S|t*Q@H?bQKcMkAY_NLd0J4e#GDIi= z06;fv&^9wsN_->10|88wSh0+E{6HdR#DE^~RII8K)T4pcZ&MTZEZv*PH-stv%y|U` z@?>Z*Dbv%+xnqHB4M1ly4Sn3GT(6V`l+4<-49k_-QMf*mb+tO}_5Y!yG4^du4;!SHSKL`*U*^u$n;~2fT!i;(D^Z0u z{9^yEkk3fuoinDl@FXMpg&xuRRPQmkuGXa@MapbhBf9um)}Yp_M`S67ap_7=&uM4rKj1P+ND6WcReAmxUG;McF`?CiV}A9PxCd3}u% z)og;9opS0_f47xZkyBXMc~UfG(vok(JUsWGw=vMPa=M1L5nFdxHTj?_fDrAY*rnA` zHiqs;V^G49WnwfsnmPe=1%t(3gf0JgOT-%cGTpw zP(7k%{10u)C%kXJK%bztF+|69s@R90EX3Ro2t?R;q!LTzEZb_le^TETv% zHfD*^M{Tcs3GMv+k{A4kP%R|2ojrP`{d%8{A_7|pc!W}i7n#BtutF4X7yW)5l5&~? ztEshgas@+hLkr9l8Yp7l?!7HmRF#q8t-D`VLMd@p3m}Q-BLq4)9S2e?2g{&!A23THbRJXZ%`j$ zEr$d<<%uo*PwqB^gLheEAEVLZkbJ$>O}J{s`xk{8YPwX_by{eNBZ`Q$UbuM9m=WId zMxQrYy&V(&UDjwt9w{=mX4L1q#0dd+<3G3&o!xtQQ46dfu7tUP0T8OcT>s^YX%A~$ zUS8$(9zBSm9BX;MNMHDTNC(ob1I%3AFC6y|8YB>8{ox&+u*yn%z=Jh3BoFG35Y`1+ z=fsJEB)R$d?FU<^XY*Xed1wU0=+z2^Uc!IUt7olPT&D9&^=v|;4L!pio=@}>G}P>W zW@PyGt}Q!%fa{w5Gx1OGHde%q zE#~lKcDee@@c;SeL&YZxC(maySk&GAz?%MD{QQrwqcq4x#MaT<;q!kse3 z+%>J32++dk=H-D-3E*cNEj;|l?4$|fCBtLdG)BUc@uf@HD3o%3RO=FuN|aYn0MA9v zPCG!9fjV{R6eY?lBBInoeC0OUJbE>rSTOg@wY59fME4*_B3L1I^AEk22 z1o*!PC5viRJUl!=Njsw3E8Ots-KYr+49dyP`3Mh(fB=j&eGLSbJDn}$>-jTgRRU%> zUYSSH?ov%*pR{5DaHOoJM;GhE^O=t?ZPMMj zMnvVUSxKDVTWYrDQL8*-QUPGwz*iyo;bLa}tSNc_CU{$wg6Mk%Mt0M{34H6)dQ`q@ z1xa=j^;jBRQ0JZ~QIfSMP^x`&3X>De`)IV>Sz%goG-RxzkcFXIArfi@Gy&d$!2 zw<01U;O;m3wKNofqNrEO)56@ev=@VmQBP!RMq%`5JUG%?hJNrGDLN4d(}=R#1+Q6E zZ(G@d2ApCfJCnskZa~@?=*$|xFaHdxl><#qZ1-p15~Rseh>D1|@Q|Oug4xcNwW`Kh zI3c5=dJPZ1YQcQU5HSt$$RK43(2TWl&M^~Kj@F+m30IE(@k5a^3iQ{iBOIa2exYqu z(0fk!X3{Z9lX-U5l<#^x?cgiDQAAw&$Gu(`gy?P^pdn2d-lapAElYuqkB{Idqo67Fe|jJ!^3p*86wNs!=(*^JW(u3R12+DEcJBthnhjEn>vX_Ao@vnd>z{!}%>r2^Jp z3w>a?IzbR<<>=~KKWsqU{}ve;s3A|8RPdw>?Y!nohC@W$IJpAtp@2qDYIZYFvRg@) zCXes$SSeS|7Zn!jwSULXo)ANqm7`ZW1^&}2KD@dG#WMC1AOT283A#xF%}79**xojZ zKAxPAQ&Z#r&F_LUPxUnGCvu;z{qxKI(;; zOkm0H%j4qVh3)_aC3)W5Xfcu%i^)ql6QFAdkm8t_nspDb$C)7-`4&)0UKSM8U%q)pT7TlJi%j+;RU>rwn*ma z5?*G!3{Eu|NQWW|dMnnmufL2VKF&voX4dldO-1KqWfj@GBocA#NA-UkW1oJHa?Y$~ zVxdeK1vCn8Fh}mVZnk0^W=BmJ7pg41Y&3GJsuT++0N^)6lIwfqfydn)R89_%wFA8w zz>5V(LQydWK<9Y6w2FcLAiEd$I}nLbuQ^r~KxaCWJb6R^Z3^ztqMrEgVyyJGA#0l( zSbSV_Cs!^!G^JBM>YqP110tMNbsdm=Oo3@hPcIDroh0UD1;LK$mibLhHAOr(qLzQ; z7;8Wh=q5^y39(kIz-WEtw`5^miL53bfllJ|kzQ1LI{K<8QY0N%F^x<>GY0KI@9yq) zc6Pw=KUUWNG9Y+f%~Fbrpv$7*Ej5c=TIXDeV@8}{HQ%>R;Q%1wl#>G*uDw%kF`%CZ zVx7dq-VVwGYu6aq*rKOTzSY&$0Stw?xjBNLcDZtCNlBclLrxATz5*<(2U1RcKG;z> z2rgMwlR#c60Hhj9BA6o(_^2vjG5zRV0ReZ4)MoJ$`neynw94(m-jeS!t4#z5azd+(IES0vKEx8k(Ej zr=Hx4YwtS_QfQmGni`9Wh79PnF!g9LkB^TJ9pGDmV+(4h!wL~eAGzW~@EZkw9CIl< zXJByf6DIs+B&=mU;YDPwEJZ08ka#(YH%C?hLc}c#A|GSL0FzbL_UifySO*Y!lq(;J zy4yqj{3b$+CFJw(G18P~_;tVSCz4+Bh*9xLaRo*0-$u4zQ+D+7gKN z8m$S;NX_hCRNA-d+;Y%oab8TXv>FJp=I?*DrH(`vnh6s5>gVXYL~9r!pflM_HkL*! z3K3%yo8||x4$1M^nOrvPk;?Y|w4b`GFK*b9Q*)6kHp?)B@P$El`sf^ifCZ>O14|vi zw@mkM>!cY5i2y?Z=-i#*i}--l9N3t!>13<$B0Kz1|5C41i45%}6x7lYK!JmCCV>QW zu8^a_>SM^KNX^#R_5$C2yF~+*Q@tzrnqYxcDYAD-bgy2&db}D__v%@{Ayzs%prs3> zP>&tz!1*Z@*oc|ava&V-(BZPy91n&YD)KA+ci0s(mNXu^GT&ONmb7hP)cnYUxe3F8 z(B*S?g~R|?V7b9Fd(d=B#bYG-ESjS_d^)_Ph(Kt^Dpy7N)q24=5S9fq0%E1hQYo;H z>#KzxtCkgbXG%liiM#&Rx3w*wV0X&Hc4}N!&oIK!Pq-Fnokc{TEF(Ys`bF5XVv|cV zoI6;w@==urv%;K+w1(vE3RSgD4O<%wXhRNO0T?!iSi=7{ND+V2TlU}4ujN8t-U4~e zRRGZgR2`s|%GxGVpDNN??_6zFO#_)55-h-ld0BQv0KZ+y$Q~p6biD~+;LGeDVeFSY zAlS;uF*2a-Sv#)CoI`z&d8mV+DTVLTVfZgBz+-Ue(diK!49}U2kBj@d@$+L0`llu= zGh1`>yfP5{`ZdYLyQy{?9Dx`22~TAw3(+_<{ENSTphubfiwXzXwEsE)z&ef;(w#tP zVJeMd0F2d@IT7?x-RT{Cgftt9+$ZMxMiqP2uE8VL#kuFE#O7>v*DT*oOgl-yO}>5n ze(;rXlikeLS$YG9Y_DAF&Q$|%QaLRT)~cxQ>efT*oY$rLQ&9qZS&Vrm`s8a)dOP-C zoimNME)~9JawQTL7dI8<&2Il~sBQPB-i&GyLNQTgslL?cz7<2qC51e&ajfX6c}s2% zb^n1wuXUD3uAsR575hGeJ9U_?jl%mf-0WI-fqz^{SNEmdh1J1~B-|%}C3By=`%WX5 z&sdl&a~aHRUA4UJTkZ4ODc+T$p?A&YVZx~S{?4s?bI92B>g@`vI#*d~LeBObelF?2 zYm?-6R{rM6@8albEFLq@Ek8wY zc1SYa0%1cwP!eF7t#GuWe+1x$m8Mri(%tp7=06;~Oe_?iwaRN%!0w^OabLu#`@JI1 zgI{PprgougQ!t@yPf1?Eg|RSd)$mZej_>0_;C!Hwc%LPydS+W|ZZGTd09B1`Pp-yf z^6W}q#xL}u`bU9vr5GaLuS$s{i{$U-!o+Z=5N){tv})3x?pz@<}n%PH#}^pR}8Kq-uq`|3_@i2qZeGrg?Oxhs{B zl$4a3df;F8Vu5iPDT+}wD{(V8T>^MFc>DAONo3pKpr7$h ziE@Srh1M54I9K(GfDPiOs2591-kj>-&yzJ-2D8TZ#Lg5x8Aa#MC^{wb=^UjpY6*u` zKr2SL$GA=A9wY*u$+? zgkcaGO@q@yME3(sR&Mz;7vIijd<$|@A{EFY(1Qnx+RFKiWMl^0zq3Kp^vh6^dw0OU zL-2E8213E$K_`KbfBa|(L^4$O?+l3<7~Twe-KYZ%ymG`s_4?~apc5=C4~$xa3+RY2 z{}^7E0K5-@H@lEB_>-tD5|3W#6;Lz=;2xM1K+y*6?C`?;MARW*qOvurRDWB8Jr&?eV_dhhqHil2 zWoBbI7p12&wh6#K0kp&-4BrGJ1C5BM3i=}L>!O6Fwh{~gf;Q-M{S9fl(ft7^ZUIS~ z1l)2ElNgR>O!RWPEo7UxZXoYE1l)adyk9= z_(y$f@M~u%t|(+73G{FPJ_X1*E51Vl`qB{%gwFYod-%?De6b-Rqg+biFH>#L9iJ1& z+Lp_+Z>3hs_Ta2!l$~!d%-wf5snXe3PGLrFVYHQUp#@H5ygdHoMR>`7(WQY%QKK*G zeZoODsupfQ$<2H7V%VSjcu*mVu2Nqctd#3Z%Ux}d3}bz*+l^p8I+|6OU3N*1S7TMm zb()7*^`jy@Uziq5BM7)SleruY7lC{6)qhV)3NlCmd~|^3^Faa6gaS3x(9NxZCk2qZ z&zAtr5vaa+4*|_~2X=JWt)}ii&@Ad=epM^>y zc6fAz9KZ))!>L@u>c-&Co1RwX&MT&ICX}(gqGRb~*@|vuFtkTmeR>h1DsNzPJnf7a zd2hvsiYr;tb!WvlsvmPYtn85#D+*eMRo~4%T-iB@{JtF740o`mmy=)ME~q#Da8|7W zr_lsT0ndg(W5pc0PpuKIx77eq$=&rFqe#TmPzV2K^f;COg&WQ}xn-Nykt^Kc6(hDx z^4P@0MZ?O@M&_7N;-}Ts)wYffR&T&%0kixK=^7}=%ScND(bk)@olIGZOo_xm%@}B5 z__5(F8=xNU-Nc$0T73<249pX{{O5nxJwg+nzV;T2;`I(9)(9~KdF4o{ZAsEOM+nQK z6(q&NjFaQA94rK{>UTmHoEz=uEDZD=MRHmooJ$!(^yokRI`ygU?AxX%;Pxh*;zqEv zbAeeoFLWmIzQ3$yObNrl3k^oq)5xoo;_FnANR09zBig$uS6+n6t+pGTE}sFmSV3*1&0^0yOO&o z7p*doKDMi@R^#C8z!%))MK|{*pyQbsPBXY!zX!SIw6c!L?Q|c{-pqC98!IP72!S_s zrZMmz>h`o)l8MZu7J$GM>HLe;4s}9SG4w98ggJT3_tf|Uk2>ymqN8>AcrrnlL{3HP zlS1f$590Vgzxb(7uL}^PL`Fvgotp?eP>+1LzX3Ah>_e%YUeNkSq?~6?uAnJ_tSlN& z7iTaMu&1M-dl6{$9Eh|%Rk{&QB=`~@b0~)R^l(QcjhKWaBRze|`4@oxg6^}_RNOzN zfJ%I;A7W-&+Tx()^n#*&(`T8-#cnwR1D^9=EBrHV+lK(78{RwtKtQBM?JM(8j&8S_roFu^)ovp;Zjgp)TUCj?fXFVq;;n_XP=+&1D> z7N}>3 zUI*s(?+4G5nMuD+iC$fH(opG{Oz9)Ti-H8`q$9=pX>%O%=ZU1S8n-*?JLg)$0NJ>| z3#z{#2puw-&x_55+B5UP@CynSYBb)1&LXGB$526#9ea7IgsNdTWyfFcs;5E+f-$kD}`^u>Ja8^mQWk_GDafbpr2ulzo` z*U*a902T}zbJIT(oK2weN!z@Zw4B20hX&jJ^RpLDi)lQm;N#d1dNBEsIj(X!zkaem zg5^f~iIh|y2wV>=RrEtr9}9CJ;cjvC(xWrtFnYdn%cgjUNCv5~l?E7K3fdM|iIW=mOHkr~tg~ z$)_MYsjI6CqNm78y15K1jxW{kjPFH()0KHc_C**eqyX<)sU5 z8jl6!l-cnRcR|F~Ir)~$6z`=Jt@U~zV697~FGnK}`0@P&-61-H-)a>rrbwC0R)zal zDpHC@QGz@ou>Fsnh>Xe|Zo$=oln<~YIy$RBkLI(?3hj zRk+DO5U8PdzC>WXGm*+F)To<#}RdGNiMYYTz{H_{okU{q2gF?aXSS|)HgK&8& z`S+dygg{8rPQl?oI|(T%20A(@p%yBYf+toYfjOLccczl2-Z7h)^cf&?;C|Hd7D>;^ z&ITJITSNTy`X6O)utr}i|AbO$%oc`@_BzYzA%5eLg29hdGm{*j(~i5Ef|APOL~`<~ zT9@Q)ivhK6M)Kg6Gn(rgvFUFjGSXqA`2!~ouV{5-I90Xxv8S#mDT(M>2ZdaIU3H4?0H7>X?NWiDI*~2D^OuN@W4P6MzZhwY4nl!p!UxP%PwD@2IUIiim z%*qxFEUeZ1iq5_zs8R&Ae*?ABPsoF7R=9YX?JG1I4WM_^>c$2JF0SkDmAM~*v+Ooz2ptVm>guOCUeC4(`dd#3R8I0;$_y}Rc9;D z5^zOg_D_GBJ7!k-8C3IQYRzT8MDPo6=2WV2y}F>ujq?tE1r>|Zm+UR+m%e(ZeP51# zLWb~1vYz9S;*MrY+Pj60He4XZ`-4s9sdECbUd4_6nbk_>rBiPAR~sNF6UT{$g>}7G zxd5tz@%{2DDxz(mY>}DdWCYq;0luL?YPcKrI#?HN>F|+4X=+oSTR_I>#<&6eB&ml8 zh*Ruk!KRgbf)p>s9`1*7Q#@J;cc{T3&eMOaVLUmw3OpH^*VKM}vbv08c$q4HZp|?a zA{(r?7fVgLioXAgTGx&xxpw!b~pe%p5AvhGKl_ z2z9vCtp}*_HZigJ}t#bkJu9ao4hjJ6ra#sS-#C17yk7701Kv2~auifY8y=f!u5*bonUE)X`y8Jqvmof;$~iEq8gi z`+t+WD5J>!~lUe_Vig`i?VN_zED4Krk^W~)BH&(Es9?|Jz1+A~fd zb)Jo2r3oQq&=DmcT-#?c><$+L0W4b>?R2h7ytvwy7OOBOi!G{eNTIB|d3S3Rn>PY1 zVTA{#4OnClYO=0!1y2U`uKK9Z@|U{RKT=J84C|U?|5*4EgR}rjpv)B_8o!q2?@%GY zp%}o^_KIA%NfRW{AKRfYJo>9_y;ImzChQgDe+%=1+6QCQ8@ ztc(2@&iig}5T^~q67uO1P_TL;NnURSXxhM4Pw4A-f0H5lYuDS7QTwf_gt#~!ryae& zJ+r`5ZFN-=-KWvd-=Ev@&*M0>&rAQbb`ewdqCTriz2$I@@*Cs+bGQWMqhF}VG#(!R zR~!zR!ouS}&nN5BI2WIt#V>Y|C( zZiD-^VknFbdDnZLA+%QieKx48m-}u_S{s!m948KY=1h2a`8zO1)E$5@nic=;4?>UIR zgEeIYc{-gY-yfd0c*2Eaf0(h8a9KYE*|jI|GSTtzopalSQl%B1yU?GKCbuzGX4>}U ziRVo;>%s_|fXAQA*vBFKg(*NIR(#Bd57Q-LNFUsWu}wuq$N~XbQp#bYpjb?c(hL=S zda%!3N~Cyv|L=}|??Sw^x0TM3!-}Y8APd_1RWz5%{N-HuYx0ymq+036*#*N<80b3H zoxRcculf1{r&kv}_P#Gj5YXg}8xp?H`PX}Am9)JS^XS1Oe*ni8h zQscjf4&-PsM{dlDj1OV>P4V#qpvRwmW12|L^_(2n_zVI`C`|$nGTnpQQi;4P>iEXq zpbltSHU^Z0x=@6eu7_`7d~ldP4vL&}-#=Uf)9b#Pl|g7GG)8(m87wO33kd~0@)Jyf z*C2#(cz7STlRw80^A8aMBcDfsD;i1dN5K}X5 z7TB?O&)G;zlFA?s-;B78{r>~*%Kufv>G)VcKr30sSCI7d9jGQ*STMX?1{cmt%t%Id z&EY+$Yx0WgFu<~Cd7QIp|GW+b_YhH57=&Hw7i9q{NKieKg674veq^S!^K9vG+C#v0 zxfk5CmBYXfB?WqG2vy*Br$9zzy`^I)|GZEBP z-~kG>=y*^dfXm+#*0?!bzz1zRgjj6z6_je+sP zgf{z87U*@?*wcsjH0xj_&cZOHih=^7nCxQy{jFyqe;puHCHXK9dD~lH&^TW!`*N9HvE%Hy zHy$@HdF*zuj?!g3hmzMGJ`JY$FNb{mdNnQk1JR&X;-z)$A2uRW*uKB6B_0aj*4N6N zt`Mr*Sh{~ZIYXqqoQ^KGfhoMzkmxdb$PA{c0cgIM@)BV8thn$T%Q#BEFUG7}g&;Pl}=d;45B zw6}3XJ4pDc$9Q zLBY1xXk41e9)zmTWBW~hJUnrfeDm#lSfm?q)d5Kdz-PStYoj=BO0IZzo2PU>1Jh#< z$~S{vmUVp6&eV6Gum)QWp))&vQ03+PJKqPu+79~kb?thq!+GPNgGwZY)Y-GOW%X&a zx=r>84mvXF0LMc1soG)79R`7Ctt5Iv2_+?;!0FGFO$B|sa;rJvPrX1UA1_YU^~|6% z8UVn#r2KWbA`Aw{O9S4&1N_=j3ET@C8+>NJ3SqU!GJuR6IA6EIr1^`~@<8_Q;2>G& zSD`ji2No7IWyl01BJXGeuX`{t-^oIh7W_TbqBvPt!s@ACTN#ld*Y5f4qJL(I??1c# zC-a^A`RE5A{D6x0S&hJf_WLO%>0;!N!E@7V2b|Q#_0uFCfkrWsGS0xz58@-sgg2MX z_n#t=S1M(H`x(=*sdD0J&y-s$n_Zm;iid?tv8*1An4+Ni`;S-E@~)}d1=Qrd$r=8bqgM8Y zHn^JV*RYAl^zet-%wferlgV-JQUAxZ>)2OcZ*F}KU^1Bal;)ds7!XB4xfJg*ESWeF z7gtLS_!%%ru%J59=SfH&Yn{FyNx&<}imdfB#Ra+G#9Pz;%eP(#yWyYFC|4Twr9eOt zsn0~M;_vAGyh*hpG93h1lQY=z@7M1Pjo{2>?~Au8cInG^)Y@;Txw)N#)NFmn677fQ zZ;g$is8*%d{xDhQh~`kg%u_}@AiD#5s>VTj-;m-D*5M=m1)45{NvEltRs@r*HB2i zIo>PKz(hG%&pbcBc4(Wqckf=+9+cUDe=0gIj>D)IROYV$^yP1PvaqEVK2=YZZuzV5 zM|(T8jQ=rP0KHrQ_&8WuZSC!+%FW|Ub~_LVAZD@D)g36*ZUl-oWs_9y@(de&z>t*|LfXtrCME2Jw)ZCyxS<>Ri|gyfVrON?B5ppNaW^c#Ev( z&)$l>h)652{x!>)m<|uRm%Wkg{F-&1joI3eeM}_aUJJB8Nv=D7Wo7$Nr#m_&bpXxa zbjdi2vC`MAWf_rB;LFHZf*+(yB~$||h0`~raL?cVDg+P{LFeDa;rrc!KA|{SB*>RG)9UfX4{fz3T>RhTJwy(zQ zv^7uidR$`FQs$U*)@4jPw6fS5OUQhDd@79kezvt`LVXn?upo~=tzX$G>k03$$% z0xQnZ^KKhQw#E;5i+#Mjis<6vDKX(29+|0X^Svznu6BNf!OY8l&8Gq8`Qb`UdkD?j7 zO$`h_B#Khi_feC-5X-|$q^t&lZeF*OQXqGM{^rHSMc{MTsR5U#_VJcr{F!Htvv-}- z?joQH6C04i@bK^qeS|V7J3D)XTd4EFdJM0|fD(jQ0~`(b3zbOF7%mG6>VmxGXYo4tB#>@v;P)No;efsoW1(dFBJ?Jg_ zzxla7g}P?x=2gW+e2|e!;Bn}{30Vogmh_s>3mjPik#@EWnK*QUa;UqD88_J*69RnXj(a8bSUCJv&{;mBI{FiHE_=5syC(O zs?8Q4Q-jE(yu93^FXaZ#2kO3ofw`+uG-e8|?8_=~3}`Yne$d?uY5+xa;=b&yaRK2n-{V7za)^Wm{(hH$e=)ZOSq(GS*1TK4rXAlk3#NN$kvFA2CsuWvh|TqGU+q&M7}!u@01v63Vm_ zf-g$s2>KViUYU#A#z%GBXS+PN)wbXz$%XjNEfynfvdYYf{+aWoGyzs;FOpNUfe$aB^H zfX6!l9d9^|&~W(C)04Ig&i+?FIq94dKd_7-nuE;71zAA?94dC!Bp!YG*L{ruQACO? zVS)DPXA%L;65EUv7mmg#>zxEcjA>I^D{k3n9Y zmev%WUd`AHjTlyB5H;XoVJVz9XBnH|daHslf>Q!ei3N~#my>6+?Up8aTro0f81s|Y z7*Z=B9V;=E2J|E$Az^`ft9CayDoPnDojp`+=3{%%Edys}4eY+wLRl*~mntyTP_~7} zpA+z9@BAQVy>s{OEPpq$BBzu+65BESSTZ25xJN^6L~hcFd89P*AE#P8nbrN@?fk7c zKxV+d|Ky{h7uvSK|9bw@hEWx_#A8{N0(9de_*!7v2Fk-jwN5tIu6Z78jN2H#RE^|w z{PPyLwZPGU0W5dgHENm#?&3-u_g$w!0c3gE;ZECTV7k6 zhR~@M<8cbq6E zGFCLj*As%lNvLbdi}AW1J#)zVFjn8TCM_;*V`Fm)$*Zlpi&!I~P70(Ys%mNgrQ#$b zdv#$1J2aeHzOxJY$dHJE?U>?Hz!s7BLpMJ5QQWX%_SnsfvTxCsV4_Xc#H%~Y#uODs zHx-utW2E1t%#jXOr?7Sz=quN3G7|o8K~Ji&)v9BlynI<_8~UQ~h2_0q^_O)&-1O$@ z*Bt?NQqEQB%TZ=bRG;j?Kmw@prZ<i-qv0+IT^zd9bK0TKd> zOBo8q%b+GJ2O&g@z~+|YNb4UZkJcK%9|%vAM43EsY~)+fw&w@yLY387kk2IT#{aWa zpo-V7pK-X9#gA3S;ZW{jcAB4l=}^|?rPFh%VLH$^Vr$V*nORq5SHf6^qm_NWF;?MI zd&ESy~#6e5_Ih{n;{-!2_lB>~(ci?xf z@|}0``T8z%eKzI}=53j26qDS_N7_+F4L2)p@=TxZj?gvxSTlDh_qdOB&6hV!dOWd7 zjEroXZ%*3V|r9`CGX{j5*T9gJLM>0gDf$Bd-c6GD5g zVp1}e3B+w3H_j+Eczx62jb!w&eYqsEQ4@O+Hdnu^G(LNEC6c@c3o-SU$L7-7V`ndd zB?+JX!P!>zIIORwhlgR$qnRIC#jAsKkZu8;o$x+;ZN4hpn_i;sY*P#eWrc9YjlqF< z-zm50RiEGCgOvKg-QE+aE7FMAas`j6T4V9l!1-btGugEzx{3m^qzkWC1v5&5^Zk8} zeTxPjxjoD5y`e69j$$I4-sYG;LJJMZ)Jf>Z*1y@McBuza&=)OPRBV}4Kg}I&@uF_b z;K(*uo28K z*^6jw=7#XpzXCY7Ymohl<)K6e@J~V#G8tuY-dhH(7T`&P zbP+xq~xD6=7k`_UE^A*i~M4xw0YI|)hlR8Q;t20 zoyP*Kd3E$O#Lfx?ep4qE%L8SM_`i?yM~kieCd`XO%l8q3@~74;@*QapBTY*fZb}y` z)sFX#?7g-%#>NMbmIMn+&&=!$FlRwQ!OkRsqyXlH0!gg~_;?at z$3l}cC$4hgXqx5N^jW7UzjcFsSxCK^hBcC z*Hd27E!G%R1nkg`mDp=q%LO`4euy;O<+UhGD2Q|jZMV%H7`sSIwMCP-uAf#+Q%e8r zSufDI_x2Vl7idBRz#WMXdjx`vn3z%MS+Sg|&B8SU{6N6n4ET)A%yJ!TAnk$996?)P zK?b)CR@f?8C63)Z`{0XQP9%fIDzKx!eT$5(UU=SYZfa5xJgcxkBjKL_$?k~>SD-^y zN2?BOgu=OSFrhJQdObZ>Xj27#ydBF-?m1Uzdy!2QhQ?$(0s=^s)J+lK3v}V{jEsIk zEK{UHh>J_r0NHLXH*(fJ;$z9G>;1E*65=$wY;yOh@0%lHpG^n*R_`$h)Cm*_@vs&A z0BdZwVX8@x$6V3R^I&``B{?xRNO+Oh*L=D+T}N#oDRSs&ctN>{ipXvbDn{=(P|?uz znKg8Wg@b5xoJ~#FVA~I&0w;QYe!f_RhPpb`T%-koCKDI_K@Hy3%`Gf6v;{)RVl!-F zxv^zr;e?P2GFv$%C1F9-#&;_)8v^tNJ2#;NXO;6la3tp_<%KSVC}$eK;w=rfdGX>y zZHP55JqH_GgmR{>vS!099elXQv$=So*7ldusy-wRUp!30!a^S>Vm{#5lz;Pqn1#R1 zUaTT|iYzyrL0P(zaf~3g>Rc{QU3X>(aW;|UB)0T}OJ8UB`yZj&osmoZoJEXfCT42d zUft|+yRr`kY(U})1&-(%ZJ2gspH<+!L`<@SoP2pX#46<z~KWvVZ$Up zL@c*AZrK1x7&7dD;Xk@ZHcRJzZ2%HY{RV8l3{45~^Cu}M=P!W5TG12%l@Nm7Hb{KM zw;&#|9{OiX8_eIbd8OnU_yIm2%6qks$gVw<=ZdAa#^e2I8?=@8{`%zK8xao<3l-Pw z3w|>o6{t~AKE>VU_<+v;58Yvzc#`iQ*}u$(nUlU@z7J=Kb&v~&@|FF!&EKK?_r93O z@Bj7e|6Bq^@b9l=NLG~dRIFr^vqGN##D(lI0MQ{texv28l}Yd?Fv7${Fv$6=82R+? zS+h*tO-DY``5Qf5?M0=Lg7R z_{(bqczChVv8}BF{7$Q-^&e)M#iN>MnX|C0{!%MGP(049PS(CA8>5-CcwbO2I*RUf zb&2_!`#qmWEy8Fzx^qy7)KFL3-yFi6L+)~v2+kH&IrXPs{1zNA!;;uSZ|7Z(5BO5s z@2zkQSJ=s^n^iwMHMX{UR4CuzJJ$5zfZG&VA2ns{^2 zetdQPB=Qz$ zIgB659&T54L{KsnBRF%Wit`SOFl8UTz8s5QEfYH#@9KW0)hqfXWLV{Up|Epc+MLhs zBYU%|9PQ5^9=>+M!>d!AQ}K$b=2#nIA|o3ZXe8o6ed~4=G+IEphDUDg^!g74$8XPdi7 zf0oi%a(@^e7S_lIun4E!2_5`oa?A;ph}d!Omz7hfP^tsDH_%Qv&!T465T7~Ikx+h!=5$S>K$PwoOEOTqp1J~(y z;n>Ps)#2u{x>~QA!}X2H3Xiz*&AQWr&%X$=>}F|rzMTch=7|kF9{H`LR%TH#S8-{# z)i>T;<`z_uWVm92Z&Fq#CE>b0Ql|{e6}-@Y^cpA0VRvWO6bUg?vlw)(EYHBZUJ5s6 z0bmL1O->eB^?G2zTQ?dhot)tf`%Qnx0tbN#dt%Qi68F;s59pTOBzV@Fm(cnLBQ@zM zxnha7org6S_htzjvpUM3;q1MLR|XaK*jV$Ew5;8TF3UdAhOE5Nfum11DU{IC0yTHK zLJxHDNUyoMiPIuX%GZ~YD%{TZafoulf&~XyI@jwmM|jxr3iV}=Lh%xgrtfKXgvuLeW_G17!K?% ze3I^u2nY`8*e0v0!jhzMhIqw-cx{*eK;AiCscl|kZw_%6E~`g!QPK97AD)gBQmqzB z$xPL#)=es>*A3l&(#eOCd7|oDo zq!mZtoBi2~DQP&<8C?uoLm&_w<`-5&hfmf(t2rqkKlsdT;%?<+#VpCUjLeG;AEDaiitH!5?)i?`XV%@;&hEt;PIhxe=O9M@L6+ z(n&?x)8x9NzHN1%i3vi>H^bvY7w*tV*SA>KKWU_!!%MXO=c5>uXe7e3Q*)F)wwfE@z0rtvWq zzlT!zWq4mLM=QDMx(}WiIuVUcfOqT^zoLP)8Xz1v8mP!$RUS$Im++mOir2c0Ff>Hbja+_Kt}ttYsG3y(3^={ z?DH;|(jxPY_ASaK{1Fr(_o98<1_u>Z@g=am1==*(4=Ubfjlmv<#z38RyS z`d71l(aRCBuy+3IP<(|Y-hoS5*D+3vw@k~JtG%5L7&ZZI34%8)L& zb_-oH{#kwt0b%W1s~+M><${xZ*;H>jM0K^@V4Yc-eajfG+{No$dDHy}NsEX>Rh0K) zyyqrPPH(SQT+|%ysX9AM=ZChANKHANiub&;WK^X z!Z7iaG>ScOF%r$RpbuJ_68Oz`z&4;0P2im9R`Q<1+U|^x6ik0*OBgl_b zi3GPcwZMGMTIC&_D5a77y5+XAwGoXsHOo8M=~;0R=7DmGVAS?2ex{zVnX19L=^9wO za(=zlyCG zA{_;eSzGh9l_8h;-VYbfPuH!1IN=Fz{iwF-_ueGg$TzSW*{_JENl8V<)LoY9CP8UX|rQ*0_#4q;-s-_Dx zUBJzQoYO-;290`V^On|$S0p|07tK-A>h;zz9mq?)vhuT8yb6jy<51gIz8tk53xDmO zcQTt}skpmbpVvOHw6bF@F6beWz(7T%US~ig-7Or1e{y0Q!|7h?(rPcA_;hQcZY!w- zM^1jA%tEK%2eSd*5WJN)OoLi(00H=@iElfU$OXw}i$%T2zfd|-EHRK2k21`3a?Z=h z;8;mTf5f1+vsp8_zqe;n`(vwiuGZAylL}pMK!8^LENydqEH`Ou2}}LdepXEfdv37w zzq_gLf=g=i(>-}59sS4b#yWOv3`2j@|0HS`~akcAR z&HdRY`3jd8S0ESK<=6O<7}EF6Z>|$hUfFfKOipUDA`QXzYHc|G!mPbmC?INVbhJC3 z>3n_EqpWu6!R`1s4jYRp2?aHLk|@WOogrB=pB)@Lb~8~wQLNgx+a-oslChlbyr$J> zFW(YZ+se1LP${M%9xHp$a@T#xG3knpWayFEJ#@DH-G)X`5lDooLV%`O;=Eue9#wE( z6>?i1^VNCFdW|$YlJL1vJgu#h+`Txv1Ola2%8J-iI+XXpwslGx6f|wYCq>J~W)@W+ zOigi$!bXCk!*wuylFwc4?Oktf7V=}_NKmVMqpDh$pZNRgYm_RM%Xl99v!-gcXhwc5T)2D%FrUqMHOg;DX@7ZVRb^1>)fYpV|0anZ3R$~*|*?IR?=n{Yw ztemu40G&9qm9Mi4u{KoW;CZt}ut{jRfa#}!YGmeYj)b_l&}BT=z@Pnd1`yRm`JNNN z8OK_b)5;g#@w5dkaBuI1`q%beQ~s}rhX}7k_z4|d=j!2aB!H_R82`|(l=x5vSU*%q zv%&+|@Vt{pj;OCjy17S(MHD>^YybBzWrpgnTm%Jh`uRo_(G@`ser0;?4!5cju{}*W za$fA~g-HDVX~z>K;m+M(!QkGtfBCbF7hHLTdV2OO=Tqf===Cvx-jm_-@H9OC1v*LQ|l;K z6(pEYv3Rs-pi#>+w8Bt~t-Si~k{K1Fv3FOr;I%v@sdHp#u5DCJve<EZTfz8Jg3#CEk;EOAkdRtX zU}>zq-RCn`br##7lWt&|la?8t^GO?*jEs!uOcT`Q(*y5TL*EWW-T5Be4*U3_B0FAoD$k<%6yLb04 zqgI`t8n7OYmUII;c0>{iAkl9{L8+N#6vQT3RqJu;Y?yMNt@UiM0BFZhQ-Al zjueHRlA_-XCWKild4>tzcYoetiukO^3p`DM9FT<5(Q)4Q$-$Xv~u8xl4BIEdw zkan*)w*GIo&`Ioecl$<02I^kA{)C;q>d=(6c2C3BtX6?$qqknN(OXGS&%$DHrSF4F zW^L`+)cLZ6_9KkZG1krA`k)UVu3!-HUCfRILK%Imy3As%>4UG_0JDzHoVb|a#4FF6 zdkb0f$7@WlfIoBoM`QFJYO(}HiN#>yD;EykxtqM^2Cv=Ug)EqSDWXdyv?}Ix7)p1v z+dNx4*Z{*gT5evWywuHMF{RLvo*ueHxvRSQ@rk#Jxu?g)S=!Ey^=19FI8N>dvU4rN zTj;AjeBp6X(t^Um-suz+7j7=?0Y+j4_>yHXhI4Ln;Hs|vF|#z>_Yr-xh@k0+Wu&1> z>04q4<^3m+lQeqYFD@<(r_4Xq4&XJ6xO4rg>6);c^Wj7t*~YHm`}diO ztTx80sPi?8D%ba7Y_{**k;@v)G9Ey=oO<+Vk&OpQmuuc!a&l{NO|te0<*TQ?17Re;75F+=20M?_Ui2wn6n-%2h_5OE z+0pauTveaXgvZBAdC>5z*paUGNl~T`PYLnc#%Cgyi%qd>xYTcMU_N~M!ZIq=^PFdk z(_zP!gPD2$HF9}{61Y$Q$d(pJ;|Mh5J74{v6XJfwjoaRc(y~wo;d39at>agt2a$ly z;&cd^H=X9uN~~R+uhI5T>70h}Dm1QN;~28e!W0f1WUoX8?aSFGGdR))Y$Ux zj96K2b@wq9y}F6c>$qHi@5jmGu*2zaq2qI4jveQ)x+j26(j0h)&|>iGk4yLIb2tf@ zEjNbOIblr)`+Q6KDR&3m()0}skUMtFS_GSgr37o2lidpIHZCbBzb*CS2|tbyvzV^h zT7Ek+G1}hrOU0BN+F?+iU}#j?hlxY(xnwCIUhFUKeO>)qd!{?Np|dlML*vv`h;MY% zE2=VT&`NT!wvIje;R7{V&`WACVP}g_xE{A->Vq4pSDk3C*v@YD>vzsberqdbI__?K^_=lW8ST3Xu!3-aL_1LhC$jX=&-{ zLudakgS;Ucff@kHOY`$%zf_d7za}PH9}T66XH-^x>g>$@)I=HcGdGd%j0RhI@7{8n zQ zVv!dWJUA#9lV91GTxn{TKR?~i8)eK{9c-T)-na}e7ZTj%j~V*<;u&+wz`Y^ zA-HP1s!Frcc7N*JCxLI0o}NBHTuf(fmdB+UGX!j>iCVQxbn^{a*h>CG*;5vJ`a?rC zHrm>K0!s|O&HCyo3=zM^N7LX&NJv+w%@5%SJ|tNQCs&Ee%>#m z#-t2P;?%B9H_H}M(W%Vw!4;+uc+e_vw6*W(uvFsPklKxk$J;yA&A`aGHM>CjYKlZu zFyz+#W800{x0aNTy?yaH$``Zy!C=Rlm~Qm1{%928u$Ih!shI?xCQRj1Zxoi&rLRVOtLyOT9r->YnJyzk=)3NCeP zyv0qAODgEE-g&pRJTn!Nfus2N?RBEDvO3!pe;qG|JT-pDqvf%(xGnpw^dt7%o2}ra zzvc_aU)A`$#%g9XTH>Q6Au%zuRslE&D(W7rV&yFH<_4ZO90d`<;NSydqH$WK9Ou*C znYjrKdHJw|jj?4-KOYw!Ak_PoK|OItj1-?;4@EjopPBkqprD+TAhYoIXMxYZ62j2$ z!^9fybSm<(F|Ibtj5G>N3`D2|g|2F3P2lAB?-0rCpx39Sp3i@#NNjlLbot>GsccqA za&kffy|N%h1kuT^sFjr;oQE*^Ffb)Y1IS}??c|W{S#N{WT6eR!vhVUP`MtsJvi`|e z9v4E;+!b_sV{fs)=Gd;zdjI6Jwg9c7+zr!V^Z6FTv&!#_T{QI*+4}dEm{LX5D;>$t zGPL~Krlu@9moez4tcC|)UNm(G#b0e{Yb(@vQ;{=b(MC(y+fb04{A04_us}WU$o*RW z2pcETPTM_Tf$m4G&dWmL!#~3tJX`3*Gz|2LoYn0x1fe|r5IU3+M3?y6jKLmhiCnDD zuWYmDmPAY!kfrF0|MCW*~?nZ-k`$I(W&OP<(S3f*|>`1Ti==OD(7>DkD^^E*? zT_@u^7dMq#;Pq;oF*3gLJA{yXl{=LK^rGCnQPxSJsoEGq>+s09{4s|z%&uoi#Ky<#=}dfeN9Tt(fl`N-G`9|C5_0UOP|ws< zWJR^$d#*WZV|Y|lMp`<*XOhwh2iNWHdZ|i^Q6ZAXS8&bJ!;fd{9=d<^KF3g5G_=||CNt42;1d!eBX)h9+>QPENS@jcBE+Q1-2+iTB_W6zuLz zU=YNSNk3J~D`;q$baPP{)J>$wN+2sEBO!=ul$_{KGc2>$>Y~abIv(LptVT~*O&JP5 zT5XYUl**ZNe3++MJfB;%&Fyfw!NfGGu48L!ztr7;hnu@Haa^&yY}VYFRT-IrCmPTo;(he=Y^o5(IOBIgW z;o)>6`N3vgocP&dTtJ=6meO6kdv}{k3=-y7czSH99u=hNhsCJY+O2KWyB*rD9khC* zjU@5WHr=CvN~QvdmA{~un24)JD? zfh2uZc7Y@|rWslbot@drtlCQ8<7{n-_@5nv)mJ=MPxMR8PpOQKseVK9+aht7o z!y=L9vG46q!@?JX_UFTC|`M{f5&n}ZB1 zl8_e{N~7}X>cqIq_;_5XNpjavh%NU=KI34|&)*%eVV1Y~lvVY!^h{m*wQT)Zd~mSA zj~`P}43enmTa;M-XXj*z4cUF3w-^`b=*bBP9tNz8TWX1+-%BlKzI!iqa}qlzjaFH+ zW4MAbPvwP>keXWeGgiCo)3LIVmd}<153OM&HJEpr|CPtaR)4is`R5+8Tzp2i=wj8Pp_ckFkRc%i57d z0YcMmu7q$2&8j9#2)V#MBUA2~JHh5;dzfBzoUZTkL9Y zUrGB|Ri$VA+U1}YOU&(%=*IP01=lqt{{XT#Z**{HFZ;9G!+3DK+q z#}jX28}%5Cbga19VlaiCtIhH^ll5DwGNi0F{9|GhplHvx+v1BWo{#^ zL_;aN#Z@+jJH`6-^l#K=?66R5ogWe%|CngIG{hE@$o$;q zH^vT4PhE%J`b^`tcvPaxo)NzH{kc0Rf4rSt1X@R15jR4}2hSe_X=e)E%P}9n2|!&U z3JNqlOg$>tp_()mAXVw9mhWl7MEQ+Iilj(_$A6j<|28}Px2xf&I22ty6zPz?2Qn`2 z@|@8!LkbS`aO;z5|ModH9V^w~MO8bhOAqa_)%48J3P8k*@=mGFpiQqo>hh_M)g6zJ zrlwlO90nK1_20+awQ1t*O`mvqso2X&$tk>RYukFi)Yei_q?0nSNAmKuH_bOQ_TTZ+ zGnZ^`78J~omS5_|ULhvtQOc{1kB;`CM!u?Gyk+#xUHu8I*RQqRgJsOS2M1Gv2rQAF zAXk+#FE1!HrKQlU`PlXtjmyTSqbsd1)p&ElKYX;rxal?x-M8dd#s};EO}MRYk^~pNmuGwV!oLgrX4NE%g4eGTWnPRn`4rlzJC(YgF9az|}D~Qjif5 z#rP8y*|(DSbBWW`TY-j_t}{Z6JU37u8ke#})zxy@4dwb~YP_kBSr@fDjk^VO}fsuEHw#3Ov3$odoij~_M11fQc2@rg}V*NMX6Vx~Z1 zteQ-&!6i2b+YYUUIsHCSNPRolFg~`HKmyaXQqcKw?ZmNa@W<99>$As_!`PZ#6z>e1Oqz*~WxQulmx!V1A`tIU;VnoX3ILZ~0&Y zm7~JE2jTrp!uP`uHT2I!;8AC&^AF5i)YbDdmhNMQ?I(G-OI*PGi6yF@>{t5Qc_w} z-VpS}I~BVeCJ=B;iAvJ{G=bi(Ogu7yXvE6|jN-LiQH4xa5$e}WPinfe4Thl?eCg1| z*0x-ZJ0v#N@7Kqx8)%hQWYA42pc1*idkS?(Y2EkBn|;)>N@WHraCF&&7X<& zk54Z03mR_TgJQ;4Vdy5a3;zFcyd6#0IY#RDyAw7B1_B;F-02?(;BhEldYB=>1%eUJ zjAX=%R7@Z|U+|3W$k?(oG%P?!|Ww5^9^|Kcg2=R*?%YV9!WRRee`@m?n6&h0qg`;YP1OOrS6l-8JLdb6XT4tRojuk!%Di0e^8td#j*mB{u3nMXB}*IYDSQ_D z?1NJ7*ZV|9OWio&R@(bR)isQ_G!suYCt?B@z&FScM2C=c2AxemLKP&EC7W(l*1-|w|@u0`@|GVJfTaNtqEg!#XIIuceNS_ubhM~-XwIlqH1{!=~E z@dRlP{$C~u90wGRN4yn0G)1F)O8JqrGj^EyiUk!W^ErH`D>Za>wYCY`FSBhzc$C$v zLeEdBpMKh$OzoE=O=(7&>&5Q0FAoy@k#>$$>dXJ?%7+jIXM^n*=d2D4=HtS3E@4GX z7$Gru=jR+Zu0wTfVL|Wh`(}8aObK{<{;>9M6C{n#m}3+u!hD;_&x9Ug?t;r+pDFUx zrjd$8SiaNIZ664ekT6=23yRRw_cJrE2oDcO$cu?}Y^;~V4{#|yoyvDY9zM5H)!3wW z-k-KqiRDB*)R~-Ik(L#UIxCwd0xVB2Um5p)^?w)UYtWVAer`0yn{RS*Y?#>p zKVd=|#4BK=X~LDM$WnH8ibF$xo)IRG#b;#+fjw9r-uSFqBBl0*Ktw1=d(?k-WrraC zztKQf!`@V7kyU|a*TF_WNXGgk!DNo|Zc9**>H*78*r3g#U=Z;n1qVl4)3@296y7+K zi;0Q;3~88-z2SN=s_G<~eMH$qT`(gw<8nyF6+=ka_AA=?Fg+Sqts>ap-{}8o>?;GR z>eh7;Q3M5~B?Rg24gpDN2|-%vM!H!lk`j`O?vNHK$whaAAl=>FtUH$9clN&f?0xRJ z{9vsaOy-isL54G-(rkfT%Wx3KPAT`ObmSzXCpwhey0*<|9S z3t*vt3xlWZOvjsOoo;KLwqQ+^bI%mGAJg}YpSgP9@VqevousC5jemSA@%103?XR`lU|n${3D2wjAl~k8Nde?q5xDOniG(2dv6T4g3Lucy>m`< z(cMi3i>xksWFWk`C9g0{7P2HDI1cLJyS{$l!E;!2+~ltQ-GC8;Ap@4GRQ3#X_4X_D_rI+6xh)}G zD7#JyS#0*Yr34uOi_WU&TEFqG?W@P`o-n7C!C01ZD?%-8)rd+g4yBaorp}jkE&d|y z?Uzxs&TeEu`PJ0sJp^#QS~6{ITcBT~R!BkL*)w$ow>%W$hg}Az2veAu{U}YxrCSdd zeYx*NIL}IPxs01K?7FL+EnJ82(wzC(x(`{NVkP%m;s39;D8f@9K& z200s{WGvWxML1vhXX5@JF%Q`JC$m6&03HSK0}j1vJ<8rie0Ml^$?_vLa_C2n zpDi78atLgu-7dyrhX|*?fD3gH?s-U1xEiDyVTFILtaAbV8 zE5v787m=X-xd{>P3n04H8x2Ei>*Bb`NW4s5v!m4oy&8V&Ig!C%71)quLbl=C8)IWh z0BH~!s8dh=j_Xa@;Bd^5adJA1X6PVfxy-He!fq!A*8-s)L^ZGQJG&-h%y-UMFg27- zX>UF8pj|=Fpdg-HN3J{U3x}-oF%2<Poe4&5!%3AAqHdj9S zz`A-_YL5ezeij|`?KhX3Oz7wyD!~txf#E_LHT;i#=!seVuRmES;*@*mF4W-jfAVO5 zAA0pv6t1wk+R9$?R>HNnkB?p9r5-tUkx>&nKI9~PQmOt24T!!=2pXQEl^5idoL+A- z2|`kYh4Y`JcEbGh12FW2AwHi10&G|U6evKfd1i-SFgQuWDp4QOIC`pQ5;WTQ3_LTX zk{i`@h`gR~ajxZM!ps^YzI{%vCdYJPArMiY#NnodBKusd46LJ5h8BcX(Ct?JaAkNo^2D|mQp6|k|X*$UizQCmK%09j}AnH4Q_|aMxn$us=e{C8UQ|hXIx(Fyp;;S;j12QY2_Je{2(m6KAHy| zhB8xdaUC9eG32U3xbeq_Zf`~_S%{w0sh3R(zLwEav!CIMj+MT12lU1woe>yZqjXPB zUh*jzL{N5K+Gz~%#<)4a_N>9CNO$y#+&|n5f)7BI4`!{8&4-i*#$;Yrc9MRK&i;kkzREyxc+)k z3qXoKlG%yhzjZk{Xc$b_+PHs}?cBd_RqJH`yTLoHz=~?+*Qo>lJ45k}4dZt&E-sem z#b{{o-GhTWN29Y;IoY4ZD|lm#?{Ap-TZ|6Iil#_$6cJ8&!IDXcx?D&^KDKSuHVl()hj0e)u7SNPI z|G!xSJ{Be8FrIc?0uJ`Z#%s#fWj>ScgZ=#vFMPFw+f*NZ!4MWWIerph4wh401~(PR zm5DA*;P?fMq`jafi1KTb$IA29jbZFhwq12yL`fEus8PFUGK7(>)#QF-_P#=J1G8K<^SJ&Svp>xzk2foVYTQl&dn`8An*q< zKsA?rke!V~LV{cR29UZ!mxDm&q5XYc;bo_1C<#dq?$xQYe(B;h)WC&{i))n3Ab$#H z;`i4#3UfX>75@2byX!ACi@%nXIQV2!eY$_2`z;b+_&VF=wtkbHm{s|6SK#V28=na< zhISRY4A=u#j&i>e4Qp@wM>f8T0@4S*d&Q*4$sh|cHKR2%%UU+hS$>eZvSy4&xOzlr zF)?g8!D*nlK#T3?H#A#sY$|Y4{upT&2o1paja-<2wFm24$ zt1bJtz5bRjT^Y9VO}QnI<_-dq6ste*^;EskeNUtV?d-Dv(k^cH2-FPqP_O|i+Z z2Js-`9-J{)dU4b^X9IIE_eM?zH<)Uu#T7$UrCm}gVt|sz!AC`T=3WJ2?YJB?MvYpDN1Cwv9g*UjEvuz?vg{f>CDqZ*$sw@?YAS~bGs_RdQkFxB?4mL0e`+X+MDEF z;6sri9c$V|j=3>EEX+j*IUhoZGfShMuaxp6=>^kVhDle9W|l()R+) z6|YaGs>@(Ed1>DyOC2hl9RyBJQ1X{#CCM>%o|n5q&4?xns&*i;@hvAbtPCuQaU+ux z2R_pyp)hia4BE<2kwo4fdC4cKX2l2L-}6d`j#ci6>=v8SVk~ElkM?^kEWhhl+)feh zc~iCYN=~{m2jk#TawJQtvM6M00mes`W46L5mBfp|q9Y*iGW&k2Asomshxi%B%t|Z%Bof_He2Pg_8YdP{c;n(jsq*I7NgMz4bqSm)(dBHnczfAPa zk9XbsBGWq)MyN?O4rht`ftW?fYl>cwcFlLe*y)CN-$>O&Ac3{%cn=tD^eV(FblkDu z2llixzY(e{87Snqf&KpR1ybmLd?@0v>#xwBugFZ#SWR-2%tbD@WiWJJNF;R=`|?3NvvMlIHQt0KKOlS z%}zwv>9YGJMD$jFfJ`V&mooX{^_vFqBM~y8`QhYssVyIpjlDB)W)JU8x`jl7zmhXA z)NXcTZNhZ-{GA|mG<0!bK~9`c9ezM8_W@C?oYUHjSM=c@6rnABF03;5}-2y4FO z9QnDcn`%lSwX$sT)`f}qy z_KT5UAVyo1Kt?KK3GUEJ@K)v6-HS+- z7Wk7gA-$Q|gA5R;3DP_@x@0{x^e5T5$SYIeI600`h=UKiZ>>U`gM7IsQN)V6y>uwA+s#mYo;5yDv zu{HIk2FsE^2|Qj+F*JXMgw)BuuuwtG*+1nx+MNb{U1US|g^eIuw@{O}%+N+X6MSpY!NT7y*oq&J5LK23IBczQteS4_N4GKsLH06MDO5+XKR0Z z%Zk9o&$auQ-@BO@zsPpa_t>eA_oePS@2W}x$=i1u%4#dDdmY(v#CIq!-R9m&z&uXF zB}%5h5zI2g-F>&q;C~;is~La4_W0O<=Ool1h5rQF2T|sCJH{4D#G2E8DH$6}a1z=V zOR##Re^{f25uptEsZ0EmWuwFQ33xhwa zQ)5SMo>qH=fr6A)vOr2VjasbvD~GH{{&}p7WEauI)+d9Qm(-|_>V8e#6;ciR%o2AS zu2@jW%<@f!4=w2ISCHBiePv1d)@!A`FTo?zZ{nWW z-Q2FI;j)f!7eW2^{iEWvrr=re1TvB>;;m+r>ficT4G9I{V3K(=y|+|7cDN07Xh;W- z0{>w=mI0-^^*C8w^)^T#7NqT)5fHSU&Zo+NKyMW!+e}t|Ib(z>=FiG}0NhO4Q$E!m zTq`e&Z*3$5h3+{%>wVrfD29Y|nP9B+>dmE@{R5O^RPTh%8H2bp91a`(dShjeu{MKZ zVipvR^cKX4$o=-n({s0n2O<%E`)Sf>i6lsuqPiVuH-XtSxA!(bjnsq0HUu|K`8*`2iJe_BA3~ zM>0-A@^IOs_4M4_vdiO;q?|ZM>d>H^=mfkf7;}U?Al@OOnLvJ&0sJKrUGZ70)YZ*@uO$CI)wORX24!oup|dj+q*VPf4}AarGd%&Pt35DqZ+p8kjKpkl?JK$POB}Q!Es4E4 ziA-q~TibDsiJuXROBU@Rws{neEoV}S1cXp;4vu>(E6%?mcaXl?H{ZON$qo%1-K<<) zZNE%lH0`q*kDjW;?_NnoMKu%Te`96_la?eMcg|5h`WzU`mOD|FR~m7BV{vgfW%aEn zyIW4~Wg3rF=im?jx`XzincCbu^*wV~Pp?-;JX>M0kLUS5DD+HeCG`Jn%%U$`lTIPb z+HbcpoGBQTr!?7JQr59&@v{^&MLf|Rdu%Wl&0!ST(gIldSh;yB zob7I?lDmtFcJ%dA1N==bz#L2EWA9c`>d0^p9OjJTfc<%?va%^{ov92pbDR3=h6cZ- zpEWfER$G0@MtI*Q1_viw&{Wi072})jN%;&>#s>#qo13&rN!j!8=t|tJdUtni)AIec zw~E>;Ev<@x4xfi#6n{hSASry>;$N`c0wr$Ng*)r!kYiA2Uu9cf@l9JNA&!TMQCwB6 zCze^j(9BH0wb)VY!^l?(-up_2rBgpDXh2T9*Y#YwBdp%z`n*I+3I-B49CCwbXw|E< zLmj)PB;4kWr#ougG^?vPF`LOB@7nDdUb<#H5n!4qBgs?{5oJ7P+NnyoXE`~gb7CBq zP;sPuA0s0gFM%)o_FN`7FzwFWz`*8~jhizJ-f2P}t^E81Eb@lvlwxyBRn+@}xw!+) zVULjB;?xyjx+eZe&#By$?(4IXRM&B?Z>Y@I(i$iP_%I>aZfq)yp2ObqbxijF+pj@E z6Hd_G#x#P%>Ee4|qFUSJm5;iK0`g*gkB#`i2U_Z*pNEGTKPHgy;w1@bI`7Xjt*%z# zmNTf`mfPwTt4`!3yKM;XIxHKwuS~ek3aIE6ZITIgeEz&RHWqe%P8r{;B6Kc#{rPneIn{fyl__E%);aGTWttL$gbbFpAUJEPtF|WNW<6 z!@5w4~ep6(kO!W7~}`XEc~_^EkV^ozu^`Q6{qhcks7V}0G1E)@xZRFq*$y}iAT zj5cv+-${~wM%w01cXxwX2mqEzhv(cyHD7)?BRDw#zto(piBV^yT6=uq=)AfbiA{CC zqty)+g+F2zO;{wElWXc*J!8ZEYTs(rtYB=+$rrg_O0JuCW1MzvIS#1Jy}h>p(6y``ttfBe<(A6lz;!QGXm3V#rvus@HW|J|4pRxCkem-#rWgNv!%dBZm_7Y2eNAE=t!1+cLj(Au!~ z+#4eiz~;}3Ai8I90`t?4ylq`s999Qu zYMkJ}XF@{6e8p2gV7J-^y4O?N6M6;~>x)0wXmir)>M&KnOcc1H*myBRl?_vLk0=&q zztj8g{rx_nY*kgcliD69uhAcl=q+D5M|Rc3SkLJ{A-?=n?LNMe`X4L{Wk0wfZ;ZJD z%o}L-O2<+M)=aHyP1(`>l$wV4$qBdlV;=6o!SvBlr`o#okJ~%7ML^Z_;TJ-?dSE`e%!&(s6<}L}#~1P2dmV6FkU>e!Q(+BUfQw74uXT$*V%FkNgjvG8T;?~WMqU| zfp7miO7h2oaB~h*aSr=Jv~=Xdg~j{(qho2y)usIx{RmS?d9A;?yp!$1X+@Mz5Yc^j z97aVs`7=VVn1%OL!FtSp`x8 z*HM7S%P4Hd5%2DfxY1Tnm~pOAs2t}gwwrMtpI zkCf}`4$e|lrLRG!!2vc671iU~UQUsw@5iJvim@@hbm*j!#YL&)4;9DDuaII#UKKU^ z2y1S>xmH4bSh3eZ+vuIUyg~vyx3^bJt^$UvtXGDH=-oYAt!$A{C|y+7^vsNMiZFvN z7qF1yn702#nBtfCTe3R~LU-91_x5Ov;+-iPdf|4oeZMiyv? zK%wJ6X$=L48ny;CHj25ry6#jmGBN@Yabr#`z*;FK<#%iGD0xi~gCVASeSIA;W?7V| zd2JzcEb#Z6i^c^y~F!vqxuV;bCC}4<3{qEoK_iI|;dL zf9+WT{8@lajGG&3X=UZ@?G5D4v6BIDz)*%%bIa)7Bd(O*gx;qwyuK}7)oW1izvp^!q&aS`bWv1`*e zToU6~EyR6j=u1eK*#(*;Mn(f}9=m5pLO&hN4*C)m6AZAO3pTcTiVOxS%HNZFHC|lK z79TaCCaHF1=Qyz#av}J^`1h#rWR(Iz0mef^qqjIIaMHJKlFXyDPDKswbhMU{vrjtu(ASAM&z{>1F}ZP6dofJufGd9D@@agRf*~>v z4$cqPJ$)Y^&P>K><<-UB4-c#rIb_%mu2!F-B`cxErE_spu109U1-P8*lOBQB(8-?; z|MC6z@A%HeXl68)6AYQPe| z5-YU!R!J=m3gvU%)dghNJK|BCN;m!}9^n0D={0YwzJI?Dh};mQ(DCtks(I()%F4>w zs$KTFMn;*rxw%8I5kU+f{Kdn~y?!j?wY3N@X20s%#=$L1^=~gGcQ;+bFiCp)^y#(o z0#WmJU7pG?7O^O0wkiX(=bkiT*a6P&@v$+b1a8?8)Q^9?<9{0muEw-mkI5hJTrecr zb6$1nCJ{Ye9m9j5;KJ-qIi38$4G$6}q*8JznXY>X5CLXm`=@lr2Ke|Cd3h%Cv81JC zWl2Kg0?!&PIvrE zG&B?lTMgBb0=oC~ZqTJAjZID4Z)PRHIz+$>59;7|ECHz~&`wEafA>kd==2l-UYtsl zi$TiemgfkI3}Q($xILn70MHo`l5dJrOuU_DDJw?$+Qmg{%tOrcMm z&k`je$%|&3WhV`BU5j@Oo9}({r;9t+(=itHEVDapL*e3a5)3dnxU+F%+~}M%2oVsR z^b4q6Sz`l!6L?Rs9)U+)jW+W$ilt5Vxm#pcNa5I6tekvkm92%v>9G8Oq-1AC25oe# z2HNQK_w@Pc>U>{R0}eh|J&AJsVXpI*LrgdC&b8{=kZbzUk$h3UDh;jDb9V<7x1Wib znaS^O0R29Y=sDu!V%4c+W@8)3S_Nhv7Z*1%F|p#|)Z`?Y=Qc-$c5-xdmY#@l=T~6Y zx5zob%FVsEC~1eQ;IM?%6B zmz8{>ehKyfVQ_p~b09_p*y>LZ;-J1Zts!(#=}*(y+KZL1c?~OVHx&p*ln?F@0pTDI zj~eYFC3*R+CKMyq{znfSt%x^sZ=3R=S1N3LrLsqA@co`g4}fJ5Ekt zXl4#|3W|@z!=3$MoSfzzm{e9e)sfTFA#(z)b7ld_REyMkv%n>9|OlhDM%}>NyB;)R%*g4iA?ds#m9#0X3fNjW#fF zpd=tQHTAS@X<=ajQ1>Np7_|eTk32mAK*r9@tgyB(2-aYyrKJhE^_sW@L0F@@3=Iuu zA#m>oc1lW0KqJ!A!vth(`x1&FcL8TPSOUuq)C)X4Js14JaiHOF7a6&n1ZS?st=A`^ z{HKp-TS?S}`+&&!S63e6ttWOvvD zgnRUN`P}fNR^(tRaH2+*OUqca4He;F0K4$f!2!o5481J1w=Mq8J6lDGn^Dm>Ck~=G z%JU-f99Kd||9HQ$t0;ci5Q2atFj;e11f)*0<)Oqksgl>I9ZpV8g@Etze18F<664F^ za*TT^0FWJ|{SS{*?d7tEf$9(~HT9F{7pO?CD7&P{002U1AbOBC*X;ZkDPPDRQogVD z$j8ZS4d=~MQ?;T7p|gn%958|sHsTshD``9I>?_-roms`xQ|d)rqdOpA6cz&%&-H*8 z6ZIs6>6VIp?;_PqcjW#J1w$QO5h0%L$^P#WJsfgwm$_wO&~{mDn1x5ik&>|E;| zq|4^9A~TI5hY{9ldtY7M$4>wlwLdsNXS>R;bmhGk92^WJDB~3f0IRl&O1ho^D{DV6 z3{9WzCE)DLS*D=ey9Xi`&)eHuV41uY0LLkyKm~-MZLp>ys{eOb6$r*9knI54MGOlc zakq?oPDl*FN~Sm8-+t6(ohvm8IQ<=eXUI+}U+Tyi+AF@w-qsOC^f*sFU$nQoTh2!k z&}=qOiRYu{ygNRLzffh+ z(uz=*L%(NX+1?T5d-wS*e|FBSe(Z5WYEzpDSRVVr-Tf)%#{Tz2T)g%>$40FDSH{!2iHUQytE*hQzPs0g?+FCSW_)YjAhRSO`Ed3(MT4!rB~L%>G`#0h}@ zlIL^_P{{%+3?Sr-=Q2C=1EWgNQIKEck2i2?jnVi6J@~ z+K{BI=eEB_|F)Q(UNQ^@2QoY;=A-eCf%Nn_Ae1gQ*z0e@U@FwnB)5VVQI z;1FW-VI`W}y*wHj)Lp!a2XwW>#Ejqh9VL6|IS>I)di*nCfeMo0y}Cw!+%l@^2sD=y zOC#%TBISV_A0JfLUFHV>Ebo7^ZN$84vAT5tS-vL&NmtiLU&^GY}h=?Cl7GP>~@< z*OHOXI;Rz};9+Hq$wz$-q&-&QpDf2fSfB^VW$5kcNfdG| z1{U8XU6v@sOX=4BE>DTJk+B$X~c*sC{a=GDojM)L=)wtU+_w9s*5n|HQ;{>lzwX zHga9Lxy#YblE%h#y`RJP2+VA}Yr{aaLpna&FH0>>lnaAt6%9Q@+ zs9=l;D%q-tdWjKaM*mDtjkWnpms;AZC0H;VQ3RA1$;%hez-g6*n8j){c+0_(Q;xj)TO|6nIcTmI(3} zz~x8+B-F+F=u+G$>F9E^vuS8(q}-AT2nc|#DA0-nZOg>Q#^+R@Ap5_1Z#oIVWNbSi ztPZZNTJQ_#cRhd530PWL4H`&saa+vPk^Oq#p|`39q<>f^`OBO3gEp_PH!?EX+T8Tu zwE;~|E@&#I&}{b;2(N?!vggsK9}(6fUIj;V>(M<)M3W;1NKv9-Q=CxYzas{eo9pv9 zIKdi)>#?fEJii!ZP%^4rjU0JEFJz3N`z*dnj7Zb;SvfL%LuwWY&=E5Ixms9D( zl^9@V20RBqctA<1A8`7kSo-_<*#KoMpo$|6wuudouNJ9GI{^XS|5;iz7_ln z49UzgM#DBSrq_2{=z7<3pGc4KU&4_X&0=c4DqB~~m}uspkmDyJ*D9p-3lwzmxOlO* zTBOK#(%AU<-8P1^&#u-&<+D}2rfGQz1WQ_4j359oiSNZO!<^p*K_=4u9iVUh0?F+e zJ{}6i10+{mFt6g)xPU?jPRIIR@RMD?Klq8H*AyneRRHYvI6ZAXAE3xPk>BXSDy$T5 zD8->VI0RgI&0hdpqDxmO$@1c-#=ri6UC-TJPeI}1mmU$tt`gW72%~sCE4kpRloUaq z`HQ#El+&N(MHd#nWTf%?#Lcs`)YbHn43ea(X2(`=^LgzbpxXhW^rRgPtMK2a7;-Lt zo$mOhr?bT+x|#%%j-Qy;tfFDF7$}vQfkOX_tC7Q%Dovma9TNlk-ZkG#E+|dL_VzeP zYN0GWN$d-al{S6=%PW-vcf2d$RU$b3E;g1Nh;2CS&2IGf`&RUj5NoGGM1Z@@K0Vc# z$w|x7={7f#;Ijo()#?Vy5&^1?j+#%iBLSqG>C4QLR~A)YSotrgT&W7;+AM#r{hxCu zptiB5`*z+@a3e=Vf+(wq1v{d;I>IyLKVaVk015rMhJc4n-9p8lfh!}Wg*zoLc`X&; z2O?JWAsxSKmz5ONW0XSBmLScCwbcKq{Qs_R%Oepf&K@ZcPN1%Y8qh=~ zBf{s{F3*(%qi!P9Mih@WKJDV)I1>sUwRCnqDT<6N>$(!@J6|l&3$oYH(0G6-fJJ&j zbE@YoFW-|ck?OUPotMXORo&lTe66Un*O%3XtBoK%(c zCJ1ecc2w8Rlby~n0SCqn>+1p-@XcC?*KC`_ZkfBhwTuiZFNh& zR!R6ikU~041q9O*HSS9nWqvv$vvFU4ZlxMP3i9ea&r)FZH8JL2xe+9y`3*OHhRnJ(5K@a?1Zb!S#qBaqPm zH42ia6}x*7xPqGFRV7O?TS!@XIVoUzoeXq6Qd&J)K?|zowLO2|8#e?jv$B9-dm;SU z(igT;4#aiIUOvCpckT<7fU?1tMP?P%c13*Z7Ki2fddyNTAPH;X!kj=*_Fw?Q%yNSD zfSPjG^(X!xJdMO792bgIuNe&QiPgOcPI z1FXd!Lb&_TTn2n?d%|&NX9vW`S308sUkKq7>ysLV52p5qI_Ag(@>TQH8DIM}Vad;- zH~o2AZqJAyn#u6T=XdVj-%mw?nGb!6=dy($Ul(eXa2KdA?9~AU(2Cp?hev3)C7M2> zHw=v<60whAH&N%G11q-p!+?wF9p)o6GD6&&@Ylclup8*O*L3xW9)~oob*}G>`0`H> x7qRtspZynn_icXhz|t~5W+~!?4-Nb9JHf4=PwqECeEf6t&2lQF=>*S5mqUtdolAitq_kC34A2MX#Vl<=3& zvW_!*DNe2^UvFDTFNcZDoQWD$$o=>zq|8yjm?KeGtK8wk1e1}yT7LiR>sl;7wT0B@ zXPD99@Otlx`>CgN_#WE&+hZazFHE)WydCy6J<@}(UHAg0YvPbX|TZ6qWZ zK+3Tuwt^Tg_oqU^SpFVI*v#?&Imaz?tfSZ^h+lxeb=0a}?TWCNttvEXU#PR&4MHLP zdHwf3Psf|cFhLbe{qpO+1O`HZP{YI>flv&qn=_-3j_0wjpKkiO-eT79%Ou}SgOZy2 zzPzYlh#DkO0TwnWEUbTE;9#zn!ItXpm_MePlO2W+hPa9yb#ybQ58oLV?*GnBEEE+@ zBuTx1%}puF5_^lwj!!Gq>L#Rexig-R!*2Ug0!L?Y6e6jhps?Bnkx1E@t+sG=Z_h8iC7&9byePgv{67-K|ih>g9;}Gz8CB ze!_#x8q2(m{_)7=!k4CRMR9-QFiJeEZ!@=#->1bv+VwBj48(+*+!=a3NfUSXd)4L6 z&j)vQ(<)sPdsHpDM(fL{53Yn5gq(7#H8!F?IrNeAnXy~(Do1u3YH)5#STy&wrpGzB zX5!D|(YvUPg(0-%Fe_|u#<^H~#B_u@nZeb+nLRNd{@9U4+K~XS{)b^~>genj=V)(P|r@G%5-Pv}AegNV@ z3i|TgYiS#jMomusd$bs=)ZOV|NQZW*BB$|K>f0{Wv2tbJfqEP&e=YE&`I|t!KET zNUWrc+C%+-ZbiL%qhp=^?6j|pT*^%tf+F+2`OAy#Un?ity^_^Y?sW+{pJ+WU)afF5 zRwHJ(rh{Ie;!VaX^LQ+}vAcauYgABGF}qzKSam6zHMo}(CHJ^|HiGETj!V8bWbwmw zF!m&UIZk1atj4-&0A9wmkT(sBrB=^|+EG{6u&sl;dcRFSMiZfkEe6lF6UBimKvb?|lnNrRzjr%($z|x>N;@2=A zr4~f&?ahj8QY8C>3UTw>)S(+I3Cu`jg_qQBbkgTlxO`b140h zVSkgkBgO0`n>V`c?R^l(8RzjJ~*uDC2wBxC-}>YyXbmLMko_~k?uA5Jfyxb=Zju=V7&{WQL2Y~@wJ)YdP16Aw;PepBFuL5yB744hH_p@ zjKid66jWqR$eBaudcQjQdAn3XTBuG=DKO`g!u5=d3_0=`8BVvA_8>Z)&hH!OaDMl< zmptxwtnNFTn=E=gsCQR$wKnbCNSMHdp`f6=xPJ;d|GJR)i?jY!m;8z>wE8y`a!D+M zG$K3Of*!p;kG*zB#%AdAtNQBfW1WxSz1p)PHgocO3gy-(&dbO7Noyn_2H-MYvszAm z$qspeM1AaG%w1KT8R@s1{>%suBQDL*xm`_6d`4&IKPN_czh_C3-uL!1Q#;}#O0s;Q z^ZMR_UKVmH-V4c9wfF&TDa&@QKA99BAD@u09d8YMG4RoOaui+dhs}>o6%}LV)w}06 zi|)5!j(*1hc!x{Yk2v#aO^78W9+5~45f0se|ZCDNW{m!kHOZAV! z_@cbgS)VpXuGhWthhZ7bdYvan;|$}2)lJ?=n5^;-on@9Q(vXEHD`ADO$_p=@^zFq2 zGrHYP)Y#AgnJ-5q19G^tGV8Xhz(9l4im{CM7u@{nbh^TM9`9_vetmTeyMsb1QFD+i zAtEx<Dk1l?X^C*IIOK=u;Dq4<<6+y_P}Zb1)xkJ{LP^ZO9ZX^O8(vO_$Ei zmPna-X=gTDMMXlQztrOO@AICHGp~_pF{+IWZ}5Cx{{2WO7)+Dq01KanPQ|o^#!}PS zvUfI(_qm(T(Oi)UV@q1G{<#ZE(uwuL<==DO)0b#^&;EU}V)XBc=RffZy!x2>k6VL+ z3WPlKyr~aRcSStDI$oWvGBd#}tu~)`UTKFlX*T~hGDi_Fy6)eHDh?(3NL*V2yBuydAFn2Nu%MB`RgRsz$w zn%(bjx}!-C=W5$lk7p`PSghBCg1&fTyLV#0e*9qL`@K~Y824eN$)qEvdK$Nz|63dp z4GAG3AyZRRdwcuMft2*<7f(YyD(M{k-&28xK!}@!F~sCkv`>Hp41%mBUa)}k)=T^K z|D*$wY25S89*;FOtRWrNYkwla>JP@#wSDrpU+cp2SUbJOb3J;-N5;i74?q`#p7m zcnvqQ1EqKsz6goWPlt|rE614yTlI@W>LRr;#y5IEkOywIES`>JCz(r?i+WpG(p^Br z^unv~J!6mrmh9F#ggkTkc{Cp}|F=}50h?M$VTLp${RXDJ*s8vryysfF$^Wk;g;&Gk zZ*o9sb#t>@Tx~Kb@exUgC~7m+g_?&a?cu^?vWN>rPLK~!qD5db?EsmW_kzrfij2$? z_pj@1;m4MAaFs4|P24$Iv?Ixw-yVU41owte@yv_dZ0rBZZn>Lw>2SP3$3xvzV>;+D zb)bDb9^oq=RnsUQQf)SzN*F$HGX1zXdz(RWozF9)pO&8? zoars=PU7cCS|UiFnYEUk{V5{Oj#GYyF1zq(m2Kx1H|&6gexPHNX?B1lId5moLbs*N z$eds@F%dICOu+dQDQOqon0kq3>#Rj{qwDnva2!*QlixAy7YV-z3kS_wKY#u_FF!v$ z;2*Wv3Qc82F~L5Qo9I7O2=y4g9OAA@=Nah(`~z=~3@#?w&#u8nrQ|3)h^F3gVEnT_Eg=L4G4 zbS~25Oj=M~gMp1v?#!BF4C8PvS@lUY+6a=D@Z)grQef!-Gc8fCu%kj&9tG8xhl9D0 zGy1_4j!EOnx@>ebG-{=?%z}bGx?%nD_wV1EO_vdKap6c_gT;5)pOFJuvVsE8f}-hi zgLYH{@Ad1v`#ed>It6u4!a&a|`y6u!-t*h%`N0EAlmn9|E1Z|O1nKq|! zzI)N55*Oyv*`1bx`lWav@!m=$F4pz4kp|MEWlG9al`67JlC=5;wd^b^qSS z!ud0aZxY$9iUyFLfSj=H^m393)aX*@%eNns^dR zrk9tO!Ah#uTCe|!Bt+(0?F^eOHyB*$2=U|7I$UTxj(fY^JG^ZxOR@6TCqi*&ep;~5 zDU~wywmj&}aX+EtBo^wX>R}PAMEmF)d?MCvV&AIO@QYK%77O2q4V+p$NJxD-+RAs? zsr4LOpjfywA6{=Z;kja-U~1!Op1iBdIO!X#TjP!@weUR?hFyZ6>x$T~zb@pN!fYGz zKveKxl{=Reex3K_ZJeA%$*PsBUDkNvU8WzSs%C`g)j&8Q8v5>jK0dfcADHNL_=spD#aE}oXXDFs;|sC zhjvkl)ay8V{#Umxy@b=FT6NoB@}X(`p&nqNO3Sux9Pa*;8ud4E5l&9!cDq?>^|5F; z_Lo6w0P#L5>RtW3`nPYTK$pXDI99fK?v8vUAmxgb)6>&}5x6+?x-f2RFzE#a1?}x$ zwwC{{H6!!?Z}!qg)31ozI|{mc%QG zRJ=O3%8m19EWiWOnfb|}&-c?>6BF;?gR<`@@UD#ULD7gedVqN_1^g z3A|aXicY3d$st&3VSC6p9QJqDn`so}b-jYIp8f$)9nFA zm;hmnNOW{`03u$vMp#(b{nctjn1D{cd?CnD@}$U)Z1b#pdV5b#Pc@vkiT_I|1;Z!? z2eYNu$fL#sfwTPT-w}o1_(Gd=C$n1dWl4iXsSqYC{Us-Y%95dbt zIl`|k-{1@XV@YX4inm<2vWDX^c65}n&Bl_d+73L1YX7{sq!k9iLoNC8H%W$qVk9=D z{ZCfI%lYP?d!YUe!QfT<=)aesjv4=R_J4i%zdz)Ed-lKI{og+1|Nid(G5}A>r4dyf zDlIRsf5+wtt(f*d^00re{*QDm@uex9KUHVT9C)kcMe=7%9eU?LlA2gfbe4nzq$5VJ zPp<0pr_Xb(t+y?D(_-+jkY=w#1`-m*e5Gm_8w@3o`L;^MBi7~C@qO9JIrPN`>oV#` zjddvSjK5;5?8{fGH*&aK1{iQNPKzz2fbWQSr^?J?A+P>}GH;N@?ScF%MCcG_{&q0# zQFeFS3TqWtZhoEb+M63*fG9NZIN~!*%tVYjMKkl+ZF)Z~x$%l{LJl&&Vod3d?n8G4 zq0v{SuR|I?HAA!lbnrXZ?ZiP z%g_EwB9~|^AE6|+mQO3&;C5!_J9O24{6$-NkkOP*-(^w$#J)T3lSa==`jrtCu7$d2 z`>wmx>m4iwjh(Bg)iTa7M>n0D2d?wOx>t7oe*PJc4z_=E_z_CQAKo9 zI{V#zm8<>mughK09O3}M67O}1&9_i>W4|yWDGx&F>$P2LbI0Q*@z_g2Io#`8P%TkGovc-pj2NM>2WdrYa*GAxume^-MJ{ zbvIf<@GxJ(Z#1BwXq+w>k{0X~+`DnYcIRdF!lV0oU2~vyVYt!ibeTHbAnCS!&dz7@ zUhYVtCC{^%p;?{Day(hx23r(s*#66+#jSrae6y6RR@=}UhPwWvBG?X2s})F)JH;JZ zs_K>TrUJ{``9KI5i_Pj#m{MXvpdGTL$L(4Cg%0-NFJz0!Ej2De7l|nzd_%jT6P~!{ z=~eOgCr6*?Z!B|O6>9c8d})0x?)XO>Mw2C*9 zbH*$dOuC*2C4(XY)_@r=O#ha<#<@*Ie<7``EGES!PFes`;=D<}YsRtDcs_7sn+>{c6?3BqvAZYZskzQAYP+)B3QcdAWy9T>iCh!x<>ng5r%XJ-!pb(odSu!-OwY3}hDmXSwYb|`ia@0!oQu#?1`0Q6 z5tY>PSL@y%1BYkdx_?u>gGU5MuZYlPj#sW#g=y0d+?=B+F5lKzij3ockP;E!A1foAY*h@!8vyJ>-whw9U#0JQ4IQB>W;hagJ_SGU|FHzV1ewU@ zG<0!u`#iIO7=C&yN8L$V4kkXVvq;wa7J0d zS;X)De}$=oDml2ttizlCY9YbHjzs5vakq;&JZZs>ojY@N*Skn6Yz;bO z8F3uXX9c=(c5~jwZ}aUY?nC|2;dSFC#T_bG>NW%FjiEsjZv1=an*!W&5{JZwy$C(o zXAYT^(#w#cF=JGBIBhsc^MBcU5ax@oUf`RZD^K%LaML_3y0eOhsXo<9yQduOI%G6~ z1$b?aeKdBG`As32bKq0M@ywqg8>}gt?c+Z-oN!9W(T9?0O*S*()3@zsMkr$&5G6**0 zZRMkH-fq61>T>Wi3T&EXkI#x+hIT*6Xce&=-2dF%dg~h>k90wJid;EGqy362CslT} z%)!>rP-Fl-g(shz;5*uIP>H@`r(6_1XD&~I# zAYPNP0d~&(f+ExL7C8}~tXa%?>z#qzAM>B=`Q2%QeaPsa2=ns&A&f%u8M za5V>H=H@M!x@z=i$K6fYWK7?vj6J}vYNoJx;qJZU-Ts7yep15VvIcphtWiOONx>;r znEYy}IfKY%?1akhVBR1*S2lJZ+tuWc=9M+8MCJb3u7%FVAsKZs53A7Fc zCrwm}*%~)OMX{@oa{XN2YFLbbq5t?6_4(x-HuL5q``sdllffpzi#+EKJ>zF9`B3hF;1(vvn&5F#J3ojc##OUlYxblQk7}zoIsAI+Beapd8%^V@ zg;BM{PI68ZkE(1S-lt)Ga4wQO0vh;6NhyPLV6NJSQ^w}DDh*NrYaGqMKnkHJ)e?O! zCJUy=D++3qF>AD+ULd}@>U5LT(L5m3_q$)cD891kSRd()Ec+wg#*FVPe^>iF`3Xue z#4i^vkHRKd7vw&8xsTiL=&W#2L?hj_^>a_N0)s(>Uf|$^@cHk***~ZAf9{PBa6^CL zMzo)6A93Q22tMZNFf^Ugd@9^N9Y>gT_qdA%@a|Kh=?Qhi%y!n5ipnq?NT3Yso zC;9ojO=>|-NA*|W`cVQ(dAIwmh?}ZQI*T;UQrJ*QD?d!+OI+Xu>10s}STDETopY|t6 z)lE~H5!2Yzw5e`RB?rlbyo#SY_KjvDtbotFw)KO1O^2^i-z9O?E`j;v*Ewb->^L&G zprvUu#Rt2Lrk&2*&tAW@IhSKj;zx=JORx3xoSuz#6#yB>W z7_+^zyX(i7zDu`cbNcsdkD$CRk>@|cuf@JkeBoUSgU=r(PN_xc+y1U9ePV-ViNNC1rQ%xBoX-=*5+1&%BA(1IT4$4%1luxQ}$QIYy8r%Qt0i8}5zIv}mX z72U*slIqqeMVii77WfYrz&}0D+C)r6fDR>XuuPej2DPBDaC~xWpP3`B_YDF90Gh9i z86P}kWq*u*a^E{V)R0l}nWh6cYh16=qlrDs^#gZt1kCDzyT2oJyg5qk2Rd^eaF`(87?HlhdQcm@=)BB`2<=_1797AGfo! zgTYS_^EYU-V8jfA!B@qN*s-Ih`KvXE;+$@f9f=GP6mmFXaSzX3OFp_;+LzDDe;4${lzCQ{etCa)o>-7Hy<1*J zMh9qVOgbf+C}b1>oxSVU&T2qG-ow2zPdEv-j+FY6Y+R|FB^`v6S5#CrZR)64JZ>yZ zRWP|rg(hlfNEXsDvU~1G;I3ns6L-#@i2fuD&)4vKmzEJ?1LG!zF!(zAjps%>g>Ud> z3oE;yvxVc0l`ECbnXaCPK;LOdWDjqY!27f{(y(rvC^F~U5K$@|fdZiK_o}gX7{AZF zy0m{c+_nU&Qg2$afMG#i!xRDJlx@ z&|YR)!7p}*Q(ZO6Gp)3%1QNl18vyG3x3{bCZD(~ap`qW!w(eo%rXxW6tnO!40mQoP zv}D?pz|g*Kc+?rEAO2|`Ek3UD`!%_L)Av)j!n-mr{GU0K2a?Fy{V9RWf(cbF?0GT=Bb+3BYHH@}{Dh4URvT!WVBMU- z2RQ!eJ?5_2ZmfBHM5ABm=f5}pmbL(GMa!E;vs_U~LCbD#5l$buuMX{?F9eP1K<{g@ssAz|)K#sWX@7QiYT5h{AvXH6E012c z3CG-ZY+vn^V}4elF8Hr=q3swgJ(-F%B2oGXY54OFZMB3>`8Rcwj{KLSWNYYvV>t7K zDN0|D&x494ld(z}%8P1k%oEdg@8uJceUL^JrpZMM>CA+yT6*|vtQ|PGXc4B(;&T_& z^ULKFl=(BRpQjH#+zjfMQ=!2h?<9?W;uC_5F3)1wvZpAz9tz?er%c$_&l^{sT)X;y zU!~&{B17G=Wxq?|ELBvMll$5ryE#w1cJz)&oH^*ruW{q!MwBZocToE|tE2r)iH1_G zAScJBPLO$WdYYe?2Zw;rekr+XMN5s_WUnqKxBjhBg5=ecrYva%LsTf~*{T5>$XYS2 zGeXa602(!L>RaKf^^Vx6=M2BU4@Katm3+iOYonp6`0+lG4k3tWZ_DwS#*!6M=;K<= z>S0({70UI+WoMSOk&)6T{jQ_Hp>0(B_RUkKEX=zkjE9E@iR#J8$coIX4dqkwYF+}C1wF7P056(G`3GK-SLj89HR zvqp%?tEq(@GUM+wx@2U(#3w>?k}r-H{+nDlmi$}6)gLig+}LX2^`t0DyS4n4mlxc- zzOvgM{$KHmxBU48SyL4JM zH~tZo*NesCSDe{&+!+6+B>x2!{x_ls&`9y_ju3RLN^FgoM=2>8ndp?1`Cqc;Rc~P# zt0#=`sdjS{BjS}@b0iq>B(!#XgO1v(Jo|Wb%zD60)<`RmkcvtN8MLwzUoI)+PV+3D`VABrS7Qj1bW;y1;1a zvkNP;v%tW>krC{? zEys-QzJ2>9HXG56f2Ym(cHB$FEjH?x|Ag=_6?@+zA*1iYbu-@TcQzT%XOqH$$ICkB807@2PPfZ z58x$5Z&j)=cS@a>NJb(U@IsLMWb6GSiX^CC-*O?{^U12I;WTBE#`W&*?1+(|7!7~| z9(Nk>?J*)*t(rv6Ah$zJ8w7a(GBPr|+WXYh_;^5!nl@!lEt=e&7vWQGLILaJ+xAVa z|0y1Xj2o|?UenRhIsTH*4+{-l16dF*4LP|;{&-zXS1EO2Ra8#MRW5qO%b(s)#&OW> z_XQ)-TIkJgW&U72v#7M6($~Q(D8#*;&B>BJ{bZC|;I;Q1;vsEL+09I=t(_`%w!zIg|4&KdZ z?u8#R5QthIlQbm_jRV528cL&mZzqN^?!~95Wdui$K)#|BNdbka!{yeMigDxqzCK$q ze6@PJWYa*i3AB(WzDK5{EU*v2fHSXVn2+O^Z$b(8xc-#~w7Dqr>F{k^{vsd%{qp5U z-var3-j6K|FfX6ykMGE*-tN{p)H^(v`b-#~0d+YkN`N5Fmg#>kBn08(w1Xy9<19=Z z+(dC!qBK8)d-DcV-J*KeLEI71BcrBr3(LIFUFqg#p^UCZpHm(oHQ&yixGLahMDbgU zg8zRB#wRl-NFAYn|}PXeeH0JuugApZ)UTZB@UFFcTe*9V2pe4B%duu_!xWc4{G?S`o8n# z#RUaoA|eEohV%*Mb2Z4gxDtEPb{zH9X0x$|wB8L34S)U!d3bod4bUl{E-Nc@rRJ0y zcXxL$RH+t#0jV-z?lxdq)~c3Ihvpm6C%|YJlq%ZS<0!dNI+0vUe+Yzj(;s~vLJoyzdzgJH9&eW{EzM&FH_Ob&;SvBlNk9@JYQzz zpBUg0e5MVZIWj<+kxy8%=o6KfN6y2;)76FS#k;gg+WA=5<-7EK$=myRbhoco{IFqP zZ*SWu9RiRHC0eg8eXcC|Bo-UCgqxcizb_<@orh=8ka0!Qx7{Y4Qwqc>V2vHx-}1-X zDNG0=VHUS7n3Le);Bt205a6Wq$Mwsn^~;xgnN#6-U&UxLeurB%V$0;CukeRZ{mwN0 za*sn+sccrMTqsY$f9vb-9~&DBLL0Kh{b)5zE*I>av-OKgLO(73?-T2}Dp7D7xf5Pq zUXBb(l(D~m^F_@0?5L9yey>80X^~h(Xft1l7SP?&{9!B|51^p8(e{7_>l^g^#f$Ns z=dR@ekc)JMwDeWFWEzYrix;oi(`p1|BvMlC>AV3$I>6swgy>y$kb1qI4gwS6E$Vg$ zf>73zy$QzooEwSs^z{60YfDMuWc`xysa0dk9QV!I@;QB z@bLZm3=v&2ZEfU-0$n=%`b0c;$-D+Z(pyiuP2C+q)YMc9R$NfNae#H@l^g< zf95*@d>MHRaOb{VtA76e(p*Bm0e*nIqTcL|0G+@rnftLn&fN$2+hwd@Hs{ka{`;@ZN=#FW=I1tnOBB4N+$8w*Sb}91k_fL15 zwvf;7Jv)DUof|$4-}DVPbEV8Q_o&Y3J)rg0hF?NLp~A` z`V4M%b#$~IX8p~2pDO1c*G_Bd4Bbxt`El{SAA~-$nyS@Ca|38$L2UZV>B?PPL9& z1FqY;MzKXEd0iQG-GhmF>gY@Re=6cDMljP{OIHhdEYR~}JImGguD>;ncQ&!WKb`z;=#} zI9@;H)!ij?ZaHcX8FyTl^p_J2N73Wypew)@8>`LUAUXP@;8V*v0eEvm_L+z;H2;EX zfcZ@vWtF{=b-Xdy&Z$SZ0cg z>)Vbt0%|bbP*WSd!&v&fJO?*bc6a7OI@wBUc-*oHKU^eAkQLANlR! zJXYQ2Q#)-t6uM8TK8z}?ecri7VS3T^-GjunWy|xF)R>cM6h3l`YDr{G$NMoULzSg| zh|onEw)4A(9*t_hfgwyaoi~! zIZKlqx5qfbFQaat8rjXW^bYV5gOih!eSKd}t_mi_W_$vZS%SG+sKJ_}_H%FKI|anz zdjaT=zAGGo8)m@|?rPi7gIB8xi0Jk$h^WYWfSw;=>|(c&O;#Nv&;?I2h}aPk?R?e$ z$mrZQ_dBu}x_)B}HJ%U;Mar)*%eWj}&zbyP<0~OyaTy7xmCi-M^G5$aoMjNm{#W0; z#X>4K8#~=aWQQ~^(ASVtRB@mvGgDh!;kDy&{rMR1i#IiDO}f7o`vIg2o2A%g`Gt9w znWo9RWG^?7}{I~kEpKw9`@ z#(|LTgtOI5ff#+QF2BrekZ(IlLH|;QBVMTrn-teEEoRCN)Z#gD^|85C( zNJB$XDH7Elj}L(A1^3c>Zcf$p^JKlK`S@oT(BmRhJV~Qm0n=6jn1HzKwubG2$c)!G zY=C<#EZp7Tbo%mnYb2rI7KQcLlT$PN*5V%FFRf-I9@J05{QDwOcS3&lNYrFOI^SB% zWE791aM$+PuJyvmw7G$-z7*Nuu`HQ>(r))5Lf8iS0*S|V$x)T?hmOl%Vv=8fl|sFV zAP$0V{`hDu3~S@U>^ozu!i&^J*TEC0l953uBmJkp z|0|4IgJWdZs-CXyl9v%5A3#bl@}eUn>HgBJ31UP5@hvQ_Me5$;@!p{?o(}A32qFyu z2l|YK-S~-PRY9iNEXW@TZ3!59-D}vBqJVuS(wsnqSZksy?lrwx>45j~-mRIMI#L(z z5pbEic(;8jp}1twe8qRHRudlKP48=Rma~Qlk2kl-&Ai}>be5>!)TqE~w9tRpe?HvU z)hiO_r`7$ebq=sag{~T*s>{Y^c=3M5O%MiF!=_BcTeWK+tHB>#+jEQta?;WsnJeO< zW=`&^F#v0+Fq@s&X65E~e|)%yNry+L&dknEWwrXHP^?ZKV+u>blxz)*(NTBT|HK6-!7h|5XSUuE1RV)Hc9)QJyl-2 zckzBx!=l!$qsBp0$vANYEA2>%2orv&ZM7WD6}M5_*-M9rUq+eVG-j-A@EGL-OnS3> zzQoRQ0A3dnO!?qAn>yE(IB{E)%iAd2#SA+!wmOyG_Sc4E>0qABcZhg54f6TKZ!@Jo zwK*z(M0)w^m7c!-iDLsGA}006rljCtWA|;IHrBVFIZ2Wezbq;!AbKAUmmnsu)IEms zw~}R8>R83y=(cB1n}9Nk7SNAjE&5UW(rE$@|A$+V`R2_?SaYQS3>TejmAk}pK?H%L z>qPotW$iRkvl0e!*W+!@Ly*=3Wh_gjU^VioK_L@>M_&XdWRITp9epDj$x-UIp2iX};vM!NmLdEkJ#y zQ-J6lKs`R8!i~uI4oULd|42*>#(b-AhiyeW?w?9+KVY$@e z`RoM}pIW^=p!0ixW&uoV*!jnIM4v#%-uCwPm9xFGv$M5znF>6Tp8f8G*AW6VDK+yIWm)J8>}c!_3X(ehmL_YD9$E3NxkdDHgsXQTXQ2+;OD?fc;yW{~MdS`-wsou<^ov+qI2>|Kt02fH02M+8Fpsfo_U( zXRgvWGxefk7~HNkvgB%Yw%Y)N)1#k0T5d~N1lGiWyj%}slXP0iZlvKZeK(i)M=qTUw^f^>No+B z9g^zYbAC0Py>9pjIifake!_!^Z7a^oz75zdwJ=B$>Xw~pPzH$A_7kBki*dtdRs{niv@^qlj<&ox%qfp zRwU3;w0b>1)_db*)QB+$w|}sL&NuTR3yBwU&C>3h$BL~!)NZthW0hx^SoozEI_s_ELI8-Y^i0@iRQLhfK6RHR;UC!k?^%bHB$irSVnnO&02*RRs_AdRqd(XL-)H{ZZn6vaB2DiC_o362s$B0!s zPRE>Zi_B+#jJ}CUJbNB};ZQw_;-(LwH%e|F&iYVAlFQDGT^aAI3tSVhr%dGoy%}V%+ZhA;R~zdYBCD)?)5kD)2iiT;(P8l!-S4i-%IGm+L7U#bL8=q9g3X`r z_%@pZb`8}fB_&l=_T5p$^L2JQIV8FEjIX4pr-iLXT)5sZR8E1c;yo(r*!Vb=Mw2Re z3`n11Swa3q+US%^MEVwuQjym2NV8H|O-?Q*Hg;ozyRY#_#e2*ZXz?NM{>8D(l-_P< zS-mK@7Fq!!39fSkjmny6PFZb6e*=2F?@9XK@k2M{^>2=o7c#c;HAm276QepuaoGmj z#?6wf>(Zr13tNhs_U)amXI;bfMmS)XM-Wt^)9)K`pF|GiMVD$n=3P;@JqqFI6s1ro zYM<}^N7^;2MIG8!YW`ZRu7Y2_%+=Y!^9*!%U}5_Ju6}e8)Ya7m<#ZERUm-_a=g1>*n)dz7cm4qZiw#cEQVawH z-Zj-_Wl>0AV@)p`Ey9uaJa=y|lXbf>74A^k6f_e?_JeA4NKb=o6v}G=akH_p_3gKD z8snS!`PY0#VMclfHe>(%E#vADY0Z)m=YB5FZC*#piA>ONPU`kI&Us*^^7_@QwJH(6 z^1OvrYMA0w40NPjP7kZZ*%ZI!Zp%aa>z`3e9sd^IlEgBU_OFc23nh&17Kp$t95&>{ zOaZ&B3t(`X7*#Hp2e2q4jwl1b%eG|-N2^{+a?C_3LKWrY{M1NmayrdkeU8x24BA8< zK!X=RnMTgOgW3Xx!`15PMt?|H7*m4SnW!P?m?Xe`nfry{ zT$2>Gt*s3wZ5AM53AWIfOt}>q8FV_*ua9-ik?aGs5a|q4Hgi0{(#Am zdSQ;6Nf#+dVzxxNx6LRnH$R*q=yJZ}r=>WU%EdrW-vG0@bc<_CsZ_QWEy=A^rYr33 z-ZXApSXc;}8Of|FL30e~iDAM(0MI!2T?B_lF0iOzzqP);4)n;#$mk1j`=b!?B>US7 zv;6VIl$875@H9x*;I{$%dCUNF1hX|%AVZ1_70&Pa`Wk41jq8G1jpcG8lL;D?GAQB- zA9j2tr=dv$`(R&Kl&!DU!1Et|6U#Gfzz?tW?ysF?Uf2AZm96q2=cc~+s4!Y8 zcQyYTJk#Eb6xs1NxSWc`lNQrupv*%=O6r0<(0WsWB}+d3>6_DXF3IL2j45 zDY`pp0RaJPYx=m}u%Mvpvr$RmMP;|`tT@26;Y+u4#UIlzFH|gj?qS08=@aZD%8LgD zfAC`$fV|P(-cBeOKE$bi4<7oPih;5u=A%1akF(wOu*S0Z=pcs!@W(uPou zy?^TrMEcU{y$rz!dJBgxX1%afR-47}{u0#tKeXEIf#~Xv_)JVOJGpOt7U7?v6ke5d&5-`?q`{^oz4|KDNC|Fqw<;2gOjvI7*7JvEqkFTU!x0SR=N zhm4m;)PNaua<9jvTeU#6U6p)s<@IYT4z6vMoAd9uFZ42*)LZJLVEOHW(y7B_|KZQ^ zib797<}IJ?dOzMyF05OA3#h1!1aM`cs0_p4_XoX>*U^Uv!f)n0xQ+|(o!W{Ekwcht zh6?phzAy1(dyG9Gdy1rNWMr!dyAm6pvi#)?fSeeqMH`>z1EhC!B;umu{=|&94WYUi zqghGm%3_0%dnI7^OAVG|l(%m8=-Q6hN(hYav`~JRr?>D}GoWXFMA%}C$v!A`!o@=hT=H9|#Vjw;7EizcWhSECPVhTwNuMY?CQr<+H$u4(h=EUKJu28?_2!md^92_J0rJ_gbo7T zCeGhP)$Y|O2pBV5uH^dZ>7M<|4{EzlE6^+by{GR9AkpBc+++_lpq>0WayEt>;!-CS z2#6V@c`r9Z9ptbYfJ-!sR^i+kdMDFm1I!%sj4tn=Ksp4Qiin8FEN$%3CHPdBb2kUc z%gIS6aO<+iL2F%0AaTdSv`xo9lwQ*Q|4&MfD|;6la>8zj#+sIlxgIY>0YfulT7gBv zTdC3DDxDFcSNN9h^l+=EyhWV}#rA8NegX~%4FT2@3*gcAjn@UDa9lvCdS1Xm5lwKo6~UKehyX|ZwurnWFxjH z|1Q91ye>iF0@8-B4RCB##bT$wNX4P*lC?Zn)lN8XKdy#8(3$*xY{KLzz4u}P=@zy> zNy|!xCvPwRztDz^*#8G@IDPw+#OwX&J7cDNJmeugNdlzg zrLajgeZqpAq*651!NCE>-};K?s&qNEeg8g7I&%WDei&`&ZZtJzsHa<J9Jv9-=qdZcaKZ_om#T5VhCYg-X%pbe+@L(RPr)MP*o?n96l> z>$4+^GG#uyP1O*wQlJz)QLz9P)#v*>}}3Y5wrK^8dQyC;kJW+Ei=-+|*D z|K^H?g?(I9vmeE9UO&`FnuDhYt-umPdNY2q{}74?&kp}T2*qU=qf`)7s+t`Nl>H81k8jYX^Fk1CfYagOid9Uemw^HfI3<7Y0LU8GZu- z0166ZtgX9u@7C7*P-W`h`|qS*KFlhR-rzxs0dFXg0rC6if@U{B!72cLSGA?a(y+WN z<$16%f!{&yg^hNdnwmO_QRC;pjLYHF(UCn#(Bu2JfR#-Q3!78(=V|*JfP>fzXPUHs zkKgX_??h*@-gwMs_mXso`^TqiV#u5;9AP?^p-~UrW)ve_S;R>!E0ox7K)b$7ZOCG& zv@ZpY|28szl34{yKr1pb@|O3$prB*jG@G!rv^4A#34wESbB9|qLD<#izl018SHaQB zaeoyG=Z{`SyI&~@qi>7Gf=I&i!v|1ubOG5RObIgAp}|4>K_c*&+5|QPbS>F%UFp@; zj?I1#L`ZP_pZ{nCzFbmMGX+Z3FEc$#ogd@>y_DYuv-~^x1Ylx<&Qkj)QVaTDJzF}? zJvUkQIcV9`-F~MT39Nv7j5H6cdFP|gEe*KH=hX@GHLk{(9i~mAM3Hu7VcvZhBDq5- zsHk(Oa@pamiH~drv0JWrkUe?wgoP!nWeyT?5l9T{`Q5kZnrS0t!a{FV(piTWr^))xws&;QN&de0R3mp!O$;^-fYiMU8-^9$! z4D~r+T9=FCrHL}5$Jl6~F{#gNz8wulW$>V9O`PVpb&_OPsB3gxaGr{-7=%h0I!mdm zD)8b9gVS6Ej2K(&&mKhBEpL5FwqGm*3v#`w@tl`m7)QU)FEc{vXyamlEnOS9V_`1xbDHU&_eXm6|+z z_AL&`g|^2@qZ8-1p`hKn;uY&H0`aw>+UUR79kyL!{4@UYkX1!*koL_xoy!>k`OUO8G_ncNU=`tr~ zl!>c$_CLs#;Wxb+IP0{Bs6LXXqR$-jXZcDcc^?*r)HBk*Ocqpa9(94 zC+4R)>(AJrlsej|nBlhD{EWRiS+xt7>0{D6t!By2=JhUp1-1k}`#-ZUNL@Z+6Y$y> z8X2VHdt;H-o6Yc_$#*4i+YkT|xE27WaWg>aL1uA&evXPsWWUl+)shSHN{G~FkX9J8 zt3V7+bIHO3w*e?KUBH-v(A-N%5G^b=))7o@V4Q!>%rwB}6%!M)CnfGDPk4pSzENNq z?fr;3wKYZ&Mqu3mPTIcmI^8{|7kuw+%$@t<*u| z#-gGVi)@O)=RZubCwI()K6$~FiM)8>*+|&NlD#8f?#Ypaa~MH6+tk&ib=4C7d52yr z+IwY5mbrf9MmLaf9g*~S-hyfFwBL^a|4dmE_uPqY7L-pKKwsK_!8Z6un$cT5K;Npp zRd6YdENzN)z1XZ~6>XI4U2!7~&qMd8byEr$zk0fA}WN^D)eWX-cHf(+i8I0c~|0 z97rPD&H~QkPD#9TybV^CTWxb;adx(Kp<0{cQ14D zh0NFv;F0n1aadN~vJ4va>zkV(3tgF=HJ@+2^L*FP&@hrtNlQ;pPfM#9w)qGYT?6D# zOJCN z&Nnb7!{ujugZuOWaSPEKEVDpAxiMWAr@Jnj3Kz4o zQUxppByM?1C8Dt}x?@;cU_)+geGiUdLMg%LUsuN4pYz92aa2oss1_Cl$te+r{|ucR zj6KkjMkUmc7Y&gZ$X@dDZxmbXU?gOFLa&J?#C|6O57~P9wGCOu2p_lPTkn`0xq%@? zYCXK^R#?gA*4Bgl{g!riL4kq%u4k3t*O)D?sv1vlh}JGtt<*c!{Z{&;+*9tL66ZOw z^V`~AIv-;b%nxUZ!@jw+wicI|sIH*E4jywo0+;c0W;6BYkZyuDS3*JpvVT^Ce#&;~ zWoe{$0I2}9b*7aXkL>^@FO>3UC|9vWuS<^AY$)dquu4#(0c9x2muN8B`%#LOA-8`1 zyxDHXWocuhiwxK5UORa`P@v$ccJF#fVxsB0cdrx_M6rKB+^>hNn*A2yTW^Xm07-a! zLdV~M9F)2{Rt_VBkEQah0tYuPlMlPubS=F~xftqrtpm&0s|}`LU8rM#*JUfLy^an* z@6mF^>8Kr1jE^Ha+uHDfq%Bo7$$g@B4rBymBd-5RM45*aU7PJ_DXtaw=us@Q-(Ym$I4*-X|B%xm zb)ElH)D(g{&zJb$pvpN;rImctjn1ZFQ*(*X^8O{n;kdUvQlO!zpa86A-?wkyW@a=T z$LpNz*f#Hkds&wx*CZ6hpe1NoljtA+3^~EwHK0mKt2}I)p5`Yf_uvJ}w5YJ~mGN68 zB?CRZFRN6XEG+WM(j0zyc`!+x1_lOJj7{NgfxX*qK&0ubNOr_CfJoL~cu9h!_@%}^ zTKZ_Xo!h{zzAk7yR@~g#xel=Sy5MI*j!i(I#`h8K^Yc18oYd6WKwQ%?rb#&InHPOE zwPtvSy>Yy|1dd)nC|(yF93152d3@{ags~5BU`RiXgJ6#wmZTu*WQA8tpPwy1OlXa<4f&Fr+ZA4_eDm?)z+!ilFEyV zQ}@2S#$hk{V$`8dlp&jG8{Yor6@g^&e%yhUIw?daS-#A)elPzNnf+-O33^TZkLwhE zy22UAMOD=D%HR7Lb0nk`H8P@Q_Z7PPJHpvt;q-S24b|L@Bg9T<8Edd;U5u8=jhC4i zAJ@~-dF6_NfrZ6LO&uaeIUmSZZIo->mY$sKo=-6K1Y{BOurCH*Napv;9MGZa8VDmV z4p<921Iz>z2=x;xv{XnZaT!!$xsXcU&=K$iO@Bu|15<8FseUAb!hRB3t0xo~_LloR z$$U1aYFd8$xEiZrn4!FJa-9Y7Ss3_EjkFi)lX&uE!t?HgDM zo4X~6wSDSC!vayT%4!L=A-Kyw zo13MgnFOn))K#t?r|@;5P82cB!Os5Z()Uw3$ZsIaf_UEE*%{AamY9?FGM;lA1Ud8i zhvVbQ&ECNGsuaExglG-g8Cc|R2*NWktXsQE`GwI5Sl>gT2})cj?&4vOx{?HvCT)Mh z#C-dggNB%Y0e^se?b^J0hP<#rk2Qnd+_j9O<71$NfqsXU8Yf4`99)!ejkq`_&F?@n zfp8WcfC;|y--k_Jj3y>33KtLW?d5E`6~?C~r8>ub*p?bDPK>0aq$DKo|L0z0TE}a> zrad>+vbVPfzMhDPD4hzl%m+I=em*|$UR^~aD7%6qR1;_}hXVOP1DrEDItl?4*ndr^ zlg?hOJEqv8&(7?~iA&MouUBbE(l4ZL{g#&*n}zQLV%~dA&82 zIe7PtXD~x*_;leQp3iah3;mlNO!99Gt50Z}(FeDRTxg~!p4?v16SU*ej>&&T^Piw# z;i9~-Z&38M)?na~lamvF($?mt^aUZGAT|_j?O?a=?JX=R0gJMr(hVqX?CtMoWP&;t zI2>M*p6Tf;F^qMR{j%uI+qaFtVlt=&_Cx>ApI@^UKziGk`kY?5lm7Kf0aq!}AP|j>jq``jK>j~cyZ?IiS_a=h)mu~B zaE5m!S#C#vG#R3t7TO=|ecIpiv7q2SfG9NK zzBXXp_=Q#Cfyui-s<9I(r>B|zRP6O%K0kbuDW7y3S2$_4?lfHaJ2P5@==}taHvzT> z9GL$nL|Xn3ws2A^@n2AA^v8q$+B~CJ^xDr@1U8%B=%iIP(KlbS1itn@X54cNr zIy~?Ui?`;94?q9&&d02&A2r;bv`Q7!yOxV=XXIx3H~f*Z-1=hoTfJskqK9;=Vr^8| z?Bd%z{AQ-7idFr92oLY*DXOpa2}TC8ngD54_th8LsrP7DN+t5W-c8gxV>!OvYo}B{ z_fkhgLgtQFr<=Yhl)RyU68MNv^N-tzleRf zhIONu@xj8d@$qjP#;La&{*N0q>Twjrbu_4OGwYy-T)?2JGLo~B-KenZYE05pFZhUK zN8E?o^D=WPYGcE~N?gw!o9BW&4)q03wB<>z?gUekDc*P6r5o2{k=-w&^d8&QMm(IU zQQ&{~^{H;|h4$wyHCi)tr%)2bygxE=cXS*|2CZ72XU;WpRWL?KXe10SR5@|fHTZES zd^XTfdNf_bEMLAMoO=6MV@F20e!FBOc6^eQv{F>jG5<2#=4kqC_tWiZT1Lu8kIKRM za~scqhVkzm5`w7_9(?xtl`PHXXa zzv9u)TcM?I(>^qHq&=M2P>y_RC@uLWpaDb?;Qk8j1pwe+s{r5E*0MJ{4K=nKp*&PU zpP;%sqTvQDDX>*sF{Ac2n)rBlV#DMtRj_M!keGom2MA^8n?RUDWCZ)@YtMi>0C^|k zSwK`h>=a{F&@rC~R)|zoK$`*S21Q444GmHRWS`xMe5Xg-+K4}Udw}j)lX?_$kP*;O z04)U7ae@k}(gbN9;Jz0ntSS^AWrsbW(yLZVA z4VRD_NlHorE)`O!0#f21Ex^wJLd93F%=cG^An}AO6Z~*hl$59)0lC6TwAop?ax^o) ztzf5|R~qQ=FDZZXemRfX?}t`9(H&m0R?6O6Z7RiSZ{`A?)EK{v(Qlc|d0V&;!I+v<+_*HIhxLpslm4k4D1H%{@~7? zBAvFo%%373eXz`=1brNUpzH2{0@UiA9bhc|#xxUn zL$Np`Jw5n#)@mJP2e|ixYL&<3#JcD$xTP+DcYGL%YPEIt*RSiqpgltSJ>M1vM+AUO z;iDWeVix+dAO{2*WuUK*%jsYppa_sn_$XShJ{P+wJ*1!jN(89p{EQ4}LKDYfehW=- zP-YfVQ=7`l$^z19dT40BgRy~92CDE7JLzE6Tn%CRtSiUdKHj=*9T)BiL6DX7r$wcy~^6GMT=B~yPH6v~V$_v0p1xKyOZp8t zn<-RDOlq8ROa{I;5Y`)J=I6$~X@t+cEzXU_E09nEG$TBUepXT13E#u;6YRO4Bx%X< zG}p?gdu%t4wkp~RUb3ZUuG_MJA<^8j+=BosD&#Y%b=A_St)hex0fS*f+N8dp^mIqh9m?DiX!(oDosLOVc$P#R zj`k+azSa{(f6Zt*pLY{QVB1yf%m+?Rxp5*r>k;(kiPWLHRrJfqlHpu6{O;pd#-?*w z$2}3niJk$R@pkFyfq~NM>WNiVTnp5QcmX$8Fuk7_jAFPnA$@#^0mcDvkGMEF5$@|x z5tY@|>XqgxP&h-(E-YMCPG$Iy)A<=aI-|%SIfcg5pcd$tJUKs{$#}ra#|PHLFY@)l zo3?vkAS?qDAU$AoJU&Nj0N$2XR#$>RC`CO*7kNyEEH_tISJxBI#bWVW#PX80YC+hf z<3YrWtGFq0YcuH!m9gD)9 zGX-ahc+U??m@0=Bdo=8Z?Dx7|^(I?d^dXJ!4|r&qiI`npu7vw;vo;K2?MhdY?*Qs! zXd#E&I{6fFRnLS*OUSkmy^xN9UT=l{{l0prE{r84rbnx}1wz8YDix;m^rYO@SsosZ zu#*6}WA^?%C&&mRxzX{MzV-E4fQ%WE&r9&B=doRH<;6~LH^u|vDkDP|S^(frq+Y&U zHKoFiBrK#N14e%^!SO7R>pUU zW9^u|;0m!YFsRV3MYK4$EVrWUI%No;Am?~oAe%`0=&2IJU0=P<=#B@YY3MhDatV-;$q=rgkWyRATU`!9Dp<6@X7*n_#lgXWr*e}2oy{(OMTdiW z<3>ze95j7PM>D0sY~bI0s1cDraNFxUm(?O3L_d7|t^^*?l}bzi1w1f39NDoDn)yG| zwqsVRszjaymK5#EtxVh*FGinWA;`(e;p3qK2GxxFz9JGWa<66j^puvb2`j^n+_Aa2 z85LM^Pn1rSw0s@10)VY{2e0G|Gw#|GoKy2~aAThJzL=EA63l z4YQx>_Q+SIu%x6*-~t}YBu-XZcozRE)NF#jRA}mf6%SiR4PcV$a>!iqV2%T>NH@VE z=Dv@O{SKH&`eWE9pa={bp$0U&goFs4F>`0RJyl??k}CwQd3bd2!6OS_|ZI7og7s)f8CD_i}t}k-28pUHA}?< zX#d@*1hFb)^esSZLNRpSLQ5;Pl%E*L6_}jpCP@NrKprYWJsev^N-Dt9vk4e#9eRVm z{BEjh#lgYB&6_vpEu^IbVH4_aWA2f(eF%&Z$U^0KOpJ}wZJ1eDbWiD?JPEd?vZL9E zk;dR!NzX)2lUME*bwC8uB!!0$!oeXK+I`tCIEd8ViD4O=m>h0SLCGLLhgA1*s@&u7 z2&MK6+dnMJWVpQHwCG1nzkLl zj)9~CiDW)N<-;K*#Qe_DWIiCFhgK(9DvNo#6f2~T+kxW*5I}m>3z(Ah90qi)4ymw! zJ_Y-Rwz=3?f_7=snffU>H3sTBD+2P&6~w&u=JxiSg4ob-3r$dd6rp3~bi5=on;Oz% zO9K)e3+w5fRbxs0*^P3Vk!iHV{>3tHTC`d-SK?{;Dq(Ab`T9~yikO(V?W1So|8=h; zR+Fc(C6-b}`K26kLZc<}w9qKqzsVcK2K?fW(4l5ztSBj=KuZM*5vrz60E=#Z0D31k zE9;dwQI+6|>bHJR8Ur7QOUKp&^8>|3yfFUOlGecCf`|vW$I{Z*1{42{sca1ptKv5? zGKvccN%^TWrsD^DJ>YkH%5rG%t-_z=86B(OyQ=rXES_7-ReylD<03^%2V$QPhvgJfG1c{9n>9(wjF zb=pE9O@iR}?c1|;{)>q$o^W0nKl#7;!1jG`Hz>Vv8wZvHv`*5g*GAI;F*yyI z6hMbif1CUL8@QI_qalVee9#b9F|sf+ngQ=soREu@Kmcwt^8(x)v&+`nNQi9b|DqtH zTUH0N2Y~qo<`3pIi1srF_1Hb00%fD$lNfS2_)0~kWBr9Uw${u|UEN5_Y~nNHRi-Dr zn%_UdVJ0~DuO6+YXkX%PpT5`slR!t~oBdkTTaX%AF2jJ``}B2$Zf~-nWF%b-oGXKP z8Ov&LadtevXu$rvH~WXVg|>rlzJ3<<#e$4$sITuS2M5;8nWaUzwlViBp83U*j9>1vh}}9JE#3FAz{bdHbhO> zg}b`oO@l}U^r$ftjFx|DIW(paDz4b60ftzf{TIY+TQ#?M1_kwi(iniwCpj`wZU|X$ z3X!;^q>*c%^ybZ<9BdHtQIReZJ6@nRX%}M8d%gpF4bba>uLxxE&`v=q%2S^Ws3#Vu zJDPgebHq7N=r85*fy{?s_VCUX9=b03iM@Mcg0sR6H4_%%Ka5vb*x@Zzl1E_-25-Jv;PYTVJa*1FZS8rH9&|-xq2)A=L2$PFAV;HMgI?2>F=k*7ufyJ z%LKz4m+y!NxM{gu*i zea5XMdIlP1W_Ck^@$cj95+grEJv$tZ{>Yl-;psmA=i5P`Of@-`&Zst-3+q4Pg}VKh z{6s;Cc7NKD{N1D2imfw>yKnrPqu>0?0d|qH9Xh=?G`yX|7Qdc{cveDLcQ7pw*dsWyY=jZh@c3_i)^TG_x5K#p42--&}!#cf>ef$+PHAN9{VR zm1tzzu5C^grNx=t4!}BLPYb2eailpMd0pZfRkec}-soH@$9iAoL&e5S#bYXJk^;wQ z{Htrm@V|A+H(q*Jf8Sx~&k#KeA4aC4Teb50NJ|xSs}dDUu8)x@Ev=5L-*t&6;+a=i z(#y-g5R~%M5D$sTGnEKVQ&m+tB$6b&LcO{<2IjBdCb2Uykt}U(jby3>jaLpYqV%+v zCh9+kikwF#51vo=vC@$6yKOSz@c#XKb2fWk`lgfUh56sM+Z6)dK|~YNQv+#M700)) z`TO~`M#;$!=yk;`^;N~?#EPJ<8)HMf7l)Cw;a669%f z#a6FvzN&B(OjSDsC~>TK7@I++vWL%5VSn_Dk{LCxF*Vx1J z<#lzbEEc$_nll3F?ejYKH8Rr;*9hW#xCmB)awZ>}BVa9r6v^_@L-%8=-AvxbXk#6q ze4VwxAq_XDcb6oHqu(7~VUJPQr>mqSCD)F3KW8~-R4frCMxW@x;g+W2GrPoVo)naLtIqj?|0d)c~%(j9jt~X zB$U|AAh}&Wr54|YV1rTYdbTuVbc*HN;Izc9IAXiK-E%FGPdmiBO#;5~phV1E}hPQ)O=Mdt!Q9+z2_ktU({OIvw5|PVO?&ioOVzvMj&5=eJ?<5&|LZ$G zi|$1X=^=CadrIJgq6(%gv!!fs`5IJ>!8>f;MB2Wh?2M!CYo**|_jr1SXtQLl>%z}K zMi#5_iXt{9)5f{%_^*;?(lCOG441=S0w|Q!m6eBUf@CJcJ|Sj@Q+5N4LU3>v{Y-*L z`6g@ZWOnw3^OpHiPJ80{Zy@d=e{n6bwW+Z+X^VpL4@jS< z$^}YR<-cSngVq6t1L50*Z;0%OmlkqM)86o~gzu48-q? z08%zbqF!T8Y8o2pmlERA;%v;U#uLL;0cprh1-q?EoAbX9^yjw%1M{e9M@sa$mja3S zOf6B`g`uHQSHmp#XDx=fI?79+VNBo)vK7eHW}k!_VB{+u}o1#M2P7Enx0JtZfX?ar~=np^6*he*!z zXK|q6do{m)Jahe8ioaFdou*HC_&FuoQ1+@1ar|OsYH2A^kw4uZRc*5o8-2f(&UGYT zox8f4!*p^TWQdNADA*sh>fgwCYBWe|RA>$~eR>Kug$S|$4sTyEO|%j3;$T76=Z`~I z$PmjQ$w;_${X5Fj^>aIhsRgk+Zx*h$^jJ6k1ft8) zSt@btH+o`Nc1x;CVroLgd`|7s0zRYn&Co?GAzsv#t1?uCIGVoI>xp$eYV8AcNCJo%o}!yK`kKO=ZLXRFS0N;s;q~iRZ7n^xBRXWl?%%Tju+Q@3%FST?`FP#UBkI zW+%G`zO|-rf3~$U63{>R%vV#c!FegUT#wwp^9f5ky#+cG!t*7oYRVdH&rx(0^Yb@0 zr)yp~oKpIr*n8W;+FAH&SX;DgZexOq@D*>H&)$0$@-)pG1D`dyynOL{vw-XQhNYIA zrIZwgRcK@X&dK4+B~^X>%+te5`F+sb_usHb*+#`8h)xK*cklG5JD~;{^@P2$@$!Q3 za%^&pUL_V9t12tq#$Jlt60RI>5mLev$rUifJ4ocI5S+dSm)_bECsSdvwX(pX_XAr4 z1G~XKVNL7p)b#M50{!EV->x|KkSMtoH=lM;?I@#Hh8XifOq5S zL4uodnIV8h4MI4WC;bs!K_AL6#lm@<1d~=4Wmnq=E+{RMDs{*Z#2J2d+`FTz;Swox3{P#;be7L zQ+6R*h*)@h{LX%?nXXa4)c+nmBkb2~>wI<$$QU*$&tSUqzYpI^dUTl*Ca$ zN@o@q+P0hHJesv`+IU38x^sTX1r^uLN(z~h_>6v5@5hCUX5aTV&kOP@X7U*hCLLBH zgqazI#P)Y;itT!C?DA25DE^*%N~8Yx!N{M9YEFOSI+90vuUC}=@aCI_X5m=XbSnO0 zn{^#yRh8mrc!;5d=y*+aG5;ToBoCmNpCqg98Z*Q@mek;Bs{vW(tu#QaF8}D_I%9MzlLZ~ zt6g7Q@9iFXuprDY60bT57{GRG5>!fFhRIe}naTXr!T{xU!O=CvO66~-2RDc&*FD~4 zku9tK{axCQryls?-F7-0ymrRV^EUxN`;b-7kAVm@G0#N(_dgI>MgJu*_@8tItkgEA znPyY9|2u2qK4v}k|9y3}S3g-{GMA1Qsw|0`7UJRpq@*S$#0WA04%S~w@Ck@jk}3N8 zw=SwJn$lNXQ-}e1UxDvv=kcFEQ}KIV(Pa6sUll;0e}sw_Fd4_kO}qw4)N z&Tr(?bh6-{D)d1E<)57FxbT=MQNZQ6eZNC7KX*hyNX&h5e;Y+eh!x|m&@g(oNms-M z&!Oo|74D4o-?6CGuHHYISHsNQaSihGYY!j9^?Ke6l1fkp^*(FJwOcw=mQCF}b(?qN zB0^xIPR|#y=NTiBu~4XOhV#A>IgzU{RqILMlf(%mVmkq`cE9WV$r%=Am7DT3yWXwa z{ufnF0(=z0WY|#kjfpGUBjve;C@!(OsI^TiEc9ua!^bC(jw5(BoJp$~EVyoBPO z;g37ma1ZlS8WbcnO^5U1DokhzS8@T9*E?-Nrxpp~?b~R(t;LOZW{)R&fBtm*Uinl< z-2O*Q=ht<|SP|Q`jiIWZ$8~SJiv+E&qEIkaf?s<(w9j7bE4~)g)RdH$Cry&k?}^RM z%s$Ts>i(S*cX11&v)-t#h7Ok zIoby-XJ3&{-t5WRvp+s1)z#b-FV1?tPkv2L-`lK1)d{0@Q>Ue)NxaY0-P-!6JHe^n zb-8^gjlYgwr8rrZkG3WB8zU6*T^pHN+$@&dTZ|nzQ_R zABFmAdzLDU)`}UeqhRDg^59GAc5&u^;|1o{2q@H7*c~r7dAjo6Ob8?dhd;#e=c%lK zDl-ZK8QDc92yKmE|t*lLoiLFgNqTbQzuf9+?{6qgE2i>y05j znK!-VAyH>gbLDijo)`{>`{`)2vh$~pqRQ}oLahUWUCzQ8xmJSW37a;qm!M=s`?zrCzQD}2Y zV&JtPi5r^f#~2e+M#QC+RTL6ZYCaKI*B@)xv;crZW$$o>5XW_~Ti|eg!vtwIspjzJ zha+GyL0gA3J5XOI<9%*1YJgfl4Fcb_T?0cs_#>as+B_#MQd1F{X0Z`4R zX?}4Z?cFyT4R<0!V?i%&qEfcAyZ@Ax_5EAotMxyRyfB~5go=7vW~3033BE8!^_aJm z>kbC9ePUSPIA*(wX+hd3IGXyc?G^2Qysk8G$zhGAgo#kA8QIvjH)kBLQQ1O0Ol=#m zT5>yIS(4}Cs)~%n8TcYi!li|$v^kY`cw#ei=FEI0pKz|%0|*Rnh)+m}{H-IZz;<)` zbY5H)4s#%E8J5p{xwF5)sJ~HvjvAD7q?F)XU0Hb_@lvJqmGNENq~L_Gi43b3AKary zGrxH-Bob#7*sk_Y!-Q7Yn_9XlUY_?AX?Esl3BsQ~))vmApL9IrsOKE~vxhA@aSwtu2%-e8-2of8}A}^B$Wp`B7-R`2E61WQ ztO|}y3>k4G@yRUr)j?7;#ad~$@tt&eq(K{%BfGIUG6;|VV=y+ZTZ8l5&%CE;pWMof zWn0Uq5Pyb>pYOZf3q@8^Qbs(JHYIV0jf=xV;4JFRXM1^7t>;osRofnK9Cl1ktKGdD z2I;Tm#rG|YP#L?G^*j5A8?YvHyJ~FD3-o*LwVVd^_YYreXBN&ds>QJOHE7!W6Rg6G zMlO|4ZGtE1?c39R*EmdL*o?RxP6m21+4C=lo13|wQakLA%q*|19iEnMHJw(}RUbIo zUzTfWnXkyr``!~Un>J&NLiEK8{oZF9#oa*yiKFR2p`nGEN`r+} zYk6=NqR_cH2apK6nQCTsc1(D<^Y4~0S6AeD`9{XYg~dg+z_SvDB!%Oom~4_?@llEh z#Ny)Oyq~$cx`#2b5up)76f$CHh)svqq5@f>B&nmcXlF0#_an^A015Dd8x97S`^X<0L@u4V)I}P!VJHX1AQ2j6ecFZI$(_?OH5m`gX zUBj8#k&Z#DNM~ndhm5eZt9U>g300+1jRu))vos=1#K-4n#g+@jn8cGv$-*s|@XE`J z-Vj00vsRio_5!1gSp?!yY%J(CiI}Y^#~>usXf)J2o}3_N7dLV47W>j{&3;v@sZK8Z z)9Cpk$Az@-$B)Y7dyZ zva*wdm8lAut-a5kg@qc-%*D|iL&5>m_i-0zs32k>-f1cw=7s^|5fh^XC{RHm5)m8~ zg@pi5AIto`gH0lbh4Ew$WP*cjuLK^aA3ykI&%V~0H}aa8nADnHk-0uO_KvnEThOT( z5+2^ED!|5;CmD5_7~hxSuP-5yvp3jy(Qwud6&&xO{nWsm4R(57HLlYg7 zlkvq?{7DULIvr;>Ja`eamZSV^JzZrcdkIb!m;7tL$KbGntl}IRYA$%c$B05Ej*j;3 zwjM?@d6C;zEMuk*f>iJb?RGci)YZ2YUzdu#mcl7s`O*LVAQ>M&%wx)C1rt}X(<5Sdw7M<@GFFhA{z7m(;& z);aBSo_ADbWi0~-aP8Ua*8nY7hDtuEsybM&F85_qG|>&Zcl@$7H*#C*RvW`iNErVi z-~^= z30q!Gt!rub^Jnz7MUsR`E+QWnB2)y(5J-rpbZg%5)|+oLJLF_$u5Atnx;%zg*z*P` z7O=FsaihA2G@-JL(uMeNeap@RBe$EMI)0f~svfH<(&g)N>MA`{d~d2r#n;^}fr|Km z@crm~P<9=03XCUcX!7#9 zo;6%_{@nm(1(A=Or_-WaDz+wXxWA!|k~>k}-1}b8fb0%DqlYWMjzdA>)aJBPq1h#Q zmGmH)&IE&8lDqH4fvrstkMZ0xlk4R2@`N3h^CdFkt+MiGQjYPWGUNoWPOqGW0o6b# zhNw7=I31Eo@DTC7cmML}a%I46YdY8I5jxiz%HUs74F4-?xE!O8+`iI`Ed{M7bsgfsbi5?wQ=W`17dRkP}-7Vceec$Y`KlET@ zqReat6Tw_k{PS}Hr_mxScL}sIa6)NcqejP&GIT?K>7R@;Ao$WyDmAEceeC1xT6glZ}a~`$HKQ8Dp8bzNN{( z_&~<+;=*C)^-&eeFZ4*a>kS;z2tn-rf-Va|L=e1GW`!oVi{G5vi{hh1 zvuC#3ie_63M;HI(l(G_^q}6^W@;YK+%pnamam3yenTlooG+F)9^f>Es^DSGj7#f55 zO)^eZ=>)zmwSSXKl=<_Q@vU1F>`UR&iZ6Kh&La})kb_!`#$zOs2I)e?e%!TiJfT%1 z-dLpP|C{fMA@8_8lJ0W5v|j2a!6N;MKvEbUcIMYup-3Vm#QX#$@UVo42c(w;QHuJ` z+W%z8h2-1E1BU}l(=EyMQHo2Sg0!}#fZ&f#FDUrb_D?ZWj3xQFKqkyyd&lpezhRZ* zCw5swxdjTVzn^QkIKZ+BG#pQZ0oDo3<^j(f`!6$WT`W%1|!r#5m;SwFWC(nlkYW%X_fKAYov|B+0g*0Y9r z>?}|Ei2#+42$LNH9==D&9p?l5*NXDE`EQXN9SgN?G2Oy{`EH5R@h^d8^@g_hLEjw)_%$y(xgKNohD7_Gd+%uGggLr!jXL2|SP3a4DzC_r&+HD;(;v$8XjfNHcz#Ho#>1ymGYQQlA|TK8qnzT zOc)qGnNCS4mi3hA-5Z=y_ow%MMkFH$(9qYLa0_~Px|IpOtFAF;AY#%!DmvlQj za1?CSP{*2xmR88ub6Q2wm9?!ud%QoH?6~cou$z50fH$8gT}dh}y;~au@ZcL%(tU$M zXXQD(ggjn7!M>G?c=qHgc!0HcIpRC7e$iU!?1a{D(!KTLumZCq5G*@5MCfoKY?jK( zM66jgb)SpnnoNqJdHwhq@UdXa4fg6FJi&NgaWpRx9m-$pFOAMBEpP)#gTfL?cn-jz zMMJlG>Y<9mH}^kMahqsfWoyGzkuKu-1WLuGQyYGMt3EzU2M4EZcdG;z+S|J%fn*{2k;Gc$LM9Xokf z$>R_ZY@VHN?Zpxlsg?y0Zox+pa!haU`rA)zg)T2!)8@WH33~!6yF@ZHfd(I&HPhQ( zWpV6NmRl zu$#}+nb1gF+(l>*3qym7^%D~hdpdWdhsTb;wUh4gRg$m3zv`?^_9W`mI%JzBggI$( zTgdU?jG8nxdkRuqT3yGyXCsBJ%Ea!SKkt)zvS%Y;112R|^NbD)zgNp4!laB;-vOo}BRb`Fq`RG>uxsE@sh+xja7lXtRFQ^t*Ya>3iZE1qI36 z5XZ>KTQ)Wo4cktBxoesj0q@20LMQ!qnpd|ZR?AnxvXghu1PoqZ`o3+Fp z9N=g(BV%KQ*U|Fy_kBJ_gtKYXo$x;VeGKbnZ~s`krJR_py}9#cR51u(%x4kk#J3*a z!6Trl@F1HWXGl#!x`y=Lh}K}Jp`mHADx}6vAY9Bxt%hWNQQ=@~tCuWxd1d>5vG<-) zQFYs!uNhDj1Ti5ZSp-2ra#Dhz0!7Y2$vNk!0+KUGE^SSTr-f_(V+sQAt59IxOPinI_b5 zY7e2}Vxz@1ES;{UBJ1Uo_wV=W>h|G5>qbg2lE?VkJ+Iun zoA7B`sqSrVUTk06r<*~`$k~swD-^?m=>tB}k00OGUs$Sts`=m{cG+kJBlX%?_7Z!8 zC)ZN|tSj~FiB zUsGj+EID_7J&?$|F){J!Sfw&^QEDt#9M{q=gAR|F`h?f9Qynx6c*|GW6_gei^Nvfv z(t^{WW=he+{3ndnPuQmGCTefZ>VCdCEB85qiA}$^*=~2~ejsVj1H}3Bbt6Z)L}jZZ z?;5eclwv85l-|90%Psi2yJv0qLzAI1E-t1le0KODp>?AwsNPK7&V_{3)B;OgR}zx( zokfzAE0R9y&qgZ;_xmz+U%srIS*!f;L2Z1T=I+*6g)sG>D%E#$auURd%`DC-D`Q_y zVwkDH5!&+gzH&1InRDRXM4&sru08Lv@%!7%jE3cQwS~Ik$d|;x_zC^;a=M!#xkW`K z-7&Va3qQ!1#&6ucvw9iNwZ=4low)YM@3#5CIQl&Z^ukxT>X*VPp7re?qbN0}Rf&lD zRq~bsby8$%UhQfzl4yUIy6N%|W-aIS@v~CFhjla6~ZyB&32?v#s2$#bH4g0b=OEylwK7>_rOVY>{BNEL&Vj(^{vTv0vz*Ajn@(q zecSVelAL{nAQ$ZMP{e6q(syp_lgc8@V?veNh-n-*xkbuPK5!inC^PXJ8;@*d{U1xd>F~}BAcOhD|s%xZ5ykHd7g}NQ-hF$kr0o)~f`Oj#t zPay~#^6Q;(S6;q+t92pX`BHy>p9h`Xd*uBbg*tWxV2buQJ_;olcmh4Vb9_ei;Jwh)vx9roErr?adtsmWycmA0<@0ZK?u z-`LWe!)jTsBFh=c6EjqEa-G*Tdlc_ez7kPtc4f8Y{B&De)7OwKlrj%@0L+{rb(9rX z34GdVXl{iDD1*p}vwrbQ234D zOM>q1qUJziTi8&}zPMPRiD(Htl}#IY{52d47ki>;F+f70(qzndIASI%m3TY>AalLoM$XJ!Ei%xlC3E}C3#Vi-W8mW47gd4=r_}Q{;lZ_GdieZ z&L63ZK>FRfY)C=T8yi=v8_A`em^j&8dn%rvdA&n+q-+!OR_4(YTo}u4M^0mfbErGCnap%u8@Kb1u>t`s6C%P14(EziyKs>x#I_w7k0UxPNFrlr{IG@8J)15u@x>YZd)==6uixN&d;tHc zs;13VR9tvI7GG3OE}vhsFua`}8+)07jh(MH3q%2Enimnu3kybckBJB_NJ^2*L?h&7 zWJ24ndKL4?xbYC_LcePk6~)%@*L5W9o5d?*UIPo@3Qv9--11Fgn_p{ zvdMs$Q{r8kAim{GH9HGF){n`^u=%C_-&8Q$I)!>)KGr>Q+03>!6cKqJKyvmBlE-Zz zH}i@s3X?DmJ;lEK@w-+j;{n7-36mw6&ex9gpC& zq%QdSwRxg;HHw*HbHefZ1(Y5tV;>e5c?!lJ$al?lI(5m&`*d=YXlpubGPyQZnsxQ| z8N-3+LLLMi9QPtwvU>+|_s^gEg@rW{!;($WP@3YfSHxlVprk?xvNDPYv&NQN4=X3~ zgN~#XXU(VI_CI>~@cpq9v_(=;=q}s zxNR#WRGcKRKHepx)H-j8g=eEu{`IJIVlN=DCt_2_$hsJ0$`IJ*S8qDzMn&a`P{IwE z*9u>)pk@e1VAat_aO&hpv)>#mb?d*x@I)|j_a*24c=ht!cy;A2`lJ9j;Lo0=IocY2 zDhabpw-l}(Z^~(C9%LiuQwJ2rw~7+q93{={nyqc6L2%?|9hW zL;tuvm!47EAo_Z*+^-F*Y&ricqJvKEc6VQC@8S;TPI|B)=9RYm)Z=@-#`w9^Fd1e?qL zou^Yd)hB0_mCx&h-mbxYB&?UQj@o|J+*}E}dCFD#N$LJORY>n->+sVVD~jc3`T7{w z^G92Yy*QA?bR)$^OM3znAJ2|1j!H5rcy>*Kv8}Cz6Dnc3sH@41+iEV6sge}SSDU=&K7gWbi zU5MHFNJYgGzY?7JeQ3S#Gc|9tVR{JaCZIYJI{x|uPvTtyGcuE-kF&ELF+Ofj&xs0` zEifExP^)%9vaS1MW_I^yPvaz+q_xW)4#;t}j^Czw@s%^K_zb*WbP`gI)^4l%- zJ|Y(sl^-!yQ&U2=vb=`QrD{h3!3(fAxZ#Pdi7E;XAtq)GmUyq%$7yU=9U{YJKhy~P zIe|>P*r(ES@hEe~%aWx0^+`7amXeY+A1Fs|KH8jFHEMEx;DjE?Q_T&yPx7{!=F-LzJAPlgyPqnu5js&_smtEoD$L-xv5rj z!L2mM&E35|ikXqmE=X0n@E#9l+YSV3)-uFa&o}<~0Tio39-21vnj_pnk+qI&j!wM~}DvuIyAUyLOYtQlaOG{~I zfwBI><>FoI;;Yv3cZrwQSlOgBIoU-%Gmt1tY?xSVEn8bZmQG74n7ny2r7D(~gVq~d zu~R6S-qpJWnNN#I5$GFJd3$S(mSKHBA}Kkd;IEC1o-IC2T5Bp_HL-9mJKSd1eyyac z`UZ}LLaj5Xat_qJ`1p^L$U_jn)I8a(?DF3q8NqpnO@`Y58z%|tZ6}A@+Ni0UDX5#> zPFT2O{Yc&@DcvtWH}?H`POi7_l~)MYvU-)zUlKV;TrME#x2;$L`qnR<=w$l?*)P}&b>+rX4)39;$ z(r=#T@+;%IO7zs4%|WI$FOD!Td&_h6S3~bjjE^Pqj1wv5*=Noy>;L(d^Os5a7vvx7 z>TeErpYsIkJw8H=sY?pM6E^(5o$yrLbQ^6~qI6hc+kYD=g0yBnl5b#`xI-OsG9fXW z5>NQ&aEm6nJ5H;wPvvDAYKP(mFPfTT0A6Q?)_=@e+1PL&uDcj6tD*%MRLY2ml^5np z1SrYLuamhybdkXK7LmCTEaFwkHIa#Q<(=(}T6PcWP0!~!b!uO4ft@k~K26J`qc&iV zQnrA3kqELqbBt@Gq01!Z#vsvitG^`oaqu7aU|Wzw55>@-_@T|KzO| z+3L7YUW0aMcU(X5oHt{orZM|L9u*k)MnIq}RoirSTx}ie8eaKM!s1;*nB8qEAW(Cj^gz1@xna#-zbt z9OVw-3W~waFPYw+DR=ymxJRIV5=VrUM0>ke>U=eO2-Q{dwi2MjT{w zh3;=g*E1W=tEsh|^xX`Uv&=Rpy$n3=M^)e(H}G zn^4ZjH}A5S#!ffGeDw3zYXU;j;lYqF!C${%R=UH7oOgnGtG~aPlBO0Jf4cTFRg>f% zhot16e)Re}Z%>4lW>h&XWept@sm`1djM zvhll9t3eHzq-@3F;yP=bYjc#skW!3;n+HX}MR`cmMYMefxO*V7*)s zLG*!eOjuZ^9)>Z6nS{6y+jrc{*fCRx0>Z6stahTnz9*4_FTA*TLf9C7`)5)cj;8J4 zaJjsTd7MBq%D8@qijLjwHRd54#LC_HKITUSM1!==gGB*1(pFdTIBCp~w-f#1TG7JP znwx`1>CDb!)_uR5qs?xAvQT~7%4%UydDIs_gqAKA-5z<~o{H;h05;sW z2w%&Kx@BLG=`YX4e|`j8Pi-HH;rRK4pBWaN{tr>-13L4snvx5Bc3#A7_%- zJ{+=M^*cx1R*d{2KWi-mBINXB$>8H0Q?H&9#dGCjw{gfF>JN=N3;PZ;GrOY@YTgwM zY(iM1e6r>_DN{@Smv$K79KShK%p02OMih3g*N#^@R(jtp%rBnpVDyW8a)@%wHNL2u z#VaaeY*h0`kBi~vff$dPQn2$(x}r39f9gn|Q)$j&E#XT(*O_!C)|uHK6zv8KtSl4S zJs3TC+LQ!} zkx}#qJ23&>-HZ8t-q;Rw$J@rI^D76@^YJ!Xv#ifpwn@i_oY>|MGGnLHFAa};!aCMQ z2JZU_6)fn$87d$I0%4P_{gI7;aEo(V=)1`763556`)d-Kmzs>W9Nr1qeN$Ib8red~ z6CpD-@)64`M!WgG6H%<1KYf9d**tU0IETJl?$jWNAI^=&-O7duLTVA0%SO(Z<-5^) z_u5TsCthH;@mg%fEfUiBxwyP9GCk(oKi$o3gHBSouD6-DZrM*_^7!e#88Pp;b6MuA zb;9&Vnx#5Pff%(g(95F;6X6J0Z?Bg=#TB?chd$Zs4zMxkJZv5rE~hE9L|hBn+jMw0 z6aB^ZW{ZjTt$f$p4^CEJ$C2x#9{(g(W7L{Q!st;zf~)tvZa!w)3VF57^5J~@wEL}^ zw~K@Ob6?zt@ibXcria&;KdFh9CF+rN1<~EnCAYJ{6tXlCPe|y>&g|-JkiCfvRq* zzR@vY`V&v6dq0Y+%Xj|Bi(YY3ae^;!BAU$6yzdL?Wm2P*G?yO|V;r*5gT1xaW*%?E zR5*7{jZdld*Q%i{bXm8LnBq?~rqIbv+@6K&PobU8gn8>+q;Ef%rdK&AC<;* z!EX=DoGV(k?oS_gj_YDfz<+z!Z8yhDYCIi(jcHz(2r8nFryk0jg9q=?X2Dx`ihr;2 z?JsH5)3G=3+7QF9&ww&BzFQNq`LXo03jUZ3rif?G?0h)+Gqqv*5TcHE7Xv0ts07-> zcfV7)fecc32ovCY#D6zQ;r$VOcFpf&d4Y+X5<>bmxv$% zp?B`%f(6d%_j$$|a`t@dSS6kY zD=PsV9SEgMxDRaCxVZ(S2@8r+wDPl-=Bn&DnwsQ)jmReyz-Rr9(}32;Pf1SR-dp_? z5O5?))9@tDcwb3reJruL%Fg_Sl9IN>_ZQxR+>Rw3oly-bQkMK)9v8{nZxa)Kvd5R+ zbbB7#WTiSbmXj0h=eGo2actCU=SN~&NS!h^5R{!I$=R|Db=roRYkK3ukqAzm*nC9KEizy0Yc6BBh~>^%NI zrG)kMkdRic^`Jmr^5mDneSMyjSFbkyiY_Y)+OevkkK-V8x50Gjob*IuySibr%AC(Z z>_4M!|#N(1$R4PBOV<_*Kj%z!`(|l(?3X^yw)T93w=?Lbzf9ERD zrICu{Ka@@oU$r~#yy>u06{jLd#IZ6g!8)zb8LyD^W~aT}_WqkFm|XG%u72gYz!FT)2d^TB{6t!(z@XEUj;h-jQu?;NKVue<~_p zK3rzdkJEWOorUGINDnK4UD;W~-Ob|Qpby_d9fU4MaYy0vBVCaI)Na{7y;Oer%o)V} zHNL9|ng4FU5W)7tk?P4$1 z?%6X4+&n}>=)mxB9h%Tv5TLtpar?^3>szE2ll8enL;R4m%a9f`%a zAl26>sA2eKWkp2kuB4VK3d{P#O8$=@jia%C_r<#T95#^-SHa7UDjh*cO6u_2pEyOG z&q6*Exjs(S+}zkIB9TzdQ*Q32*1SFMUCH&7jpoM>JZ3pnv1UKAYDdg`Va=lcVAm`Q zhOJ@#bwWZWquU%devCSr6gZd3osW2>Qv3RZ$U&5D?$d~iYmIZ)9Hy*2#rM9z@{~`U zzfU8JI%RDmIvQ)Y+@PNVl`g&HoKllt_x7Xg zhzPw-(YiT}*OL}awM5Ck>a*0Ir)1q{x%CV@aT>?sj7FNJ1x!r1G&HY(6^MuYRp_K= z%YKpSQMebCmV!G@WkF5A-NKR*`-V>;o71&da5%IK4XHz`#avFm?2q$o?GDaV?>|fF z|8BNkI>8?rqUP&&EU9f@q6cKhm7Xg%@1d8~g;`iIhH&i@=+6z8m_#Wny974RSALO0 zr?bb9Q`6XQjhMp7gyol{9ggBHtbxAgp9T5v-fdkO*7`x-)n8#wp;#7lo4EJ<>PloZ`>!n{JQSqdU3lGj?h3wf7 z;6J}c;$TP6PcKPx-9UbZvC^VHGbS?Qs_i1d<{ou$uol`0$stJX2rk|$Zg|(kB7-X4 z3A{2kW>(`pQ<0~5*7x>%8tv6u7bE@hu0{6N}Wd#}j$T^)) z?;?-(N!uQh?X+f*`TOHGJc0YM%ga9%-6f8C;TD-jUGqH3Z*xa0Pfgp;FS3HOxxp>C ztq5DDrPB}Wv|d>FNJU2Rv>xXw`$a{4dfwlVNu*)YjlcDq)V1LarKrGps`#u-!g?z! z?gMpHY~P_wS2$8kRLo0B)EX!Aex8bG|7ezuxAaQ`{v`vocyYR_rM}`|gHGx6NO^Vb z+n>yBTof1#K4*kt7t?Kp#YsaSmGXUsJ-yIM|8pw5^y}6q&z90^2j>s+`0p969Hmdb z0tdw^=gP$7=^B6kBJ^DU-}3O^=EbNgdnjIs#=KNg_6`sKR76kT=#H?|Elyw;9;7A=l3f^lPuGWN+s9o&-gzjY*Pyo*Gxc(q`A3i zE&eRHcyXPYrdCaUkV*UXU9y>R9`f*z0lBR}*W2`O@@qIDV+~+qE7e!JCS-wL42kr( zNXvNZM&y6x*@b!YRWBe~g3Z^}59HUQ_~LVvh3C4d3_(_l6SKi# zL&A2Rn2oZ-h*DC1r7_ij*O&?84~Oj9wVCvIc7H+ML$7CS(i02ZTgx$~A+sIj#5I|j z+4c>>nTV4Zg<`=zcRL?i%J_el8U@c^!*lp71Fh9-AZu-)<0&kqdz;CU8m7&aUqa;M z*bUq&qS-p$oZb?ZNgH$D4K#dw$hmBSY{$uL*ESlXnW%sxK*fRDJwCPtN15f{59&Iw zo)3DQ+~08Dca(|evOi33Xlgl?b~=8(xp`iwWiKL@U1V(iMR-X4&T7ZbVgn`RcRv1f zlYWVPX1Mv53n~NAQMtOMPPdI}v3BPpZ#Q@P#aQ=0=Z_4hyCW?L>@ zIhSWjpFiE_Yp(xTZsT82q=nw+%RQf&IoS8a#KbR?mXHvtp*+|*8E;3&qSUHy+uEij zePj%BU~hObQp!xwMxn1-+1weqneh3*mT9!2>B1!@R@T7Al(cTNEXvhdHvQl`3^gT| z0t4?197*3-VqOk*LKml(+9RWxBqamyb#+BgSwf`LiB!zqk$lKA~7X5w}=z>el#7+#^&M*<9@Si802oa(rcRi+f^~(OTCtuXz%10cY zHfk4}@kX1>JZz9c=ijSU{IafTiRd?xky@}3%ar~6o#^ay`2$|^Kl%qW4CV`-i1`L- z1LhkD?dK6>8;XCR|473` zd~{SqtgGR9u$Q3Fb#M2wNOq@)%{tWi(h+&}Sh=cZCY+0$|^ zNXWO0R;_$%Be8>Nc%-C6ht$bTuim}yD)DOr!FRo8A?}GOPw&S%yW$()6aWQe1 z>byLlr$6Ht5ylD)vodH4d3qO*lq~=>JWM>12WGdh$S?~3ul0*_6O;d@KB#vc4sDlX z=6v`5t$)gQei0p=-Gv_~BqX`d?uUm7j@x@W?OeaZ;**lbi)*U)U|^uJ$s;9wo5W>K z6yE@oTYf=)LQ?bAu=9?`5N1>r3v6z0_k>EPW3$ZtDt4d$Tf+y+D%^gLDY(7AkMmbL zj!9VIF{Z;|zWuG;|FO;h5AEs&$@~71{wAb+*>Jsoe?p@Pb3o{{3sc$oJ`skY7rw-O1ULjxo(6JG(L_TsAaV(s^FI zGC%*#%Vy;aUHFg-_FpE>W0N)3z4&UiIP~$OZD^=^ezsv06O}*y&mVP^RB=jQwQy;^ zVuGG4Wn*tBB~54vEA*u&X6<1`6&0VMWWK#Up;}>;#QxD2ghev7%}m~WUY4fFM{GaNi@m`--O zVgP@6uS>WVF{nvG%c1U2Bgb-lQ5S3dX?MMRC$3us=7`@M4u_>3YHU(Z>9Qsv*CVX? zs~f@_{Rp+TvH5Z;NG=Jm8r}Jmfq}|nPn9w=I@Xf!UB;dwJ8!u>|Cb$!6Ek^2N1E4j zO@Ia+ABQ_VSDOzx4LX1Qnu95+Iq;N$awaG@H%fGmwSjwRG4bF-@jvzSw&#a`Glxl{ z;Fz$o<`RhXd$$%7rTCRECK@g_gL+O{vz3*o%VyKrkc+HM}pNG5SVz%dIa}>;rs+}(`VM;zQWY=>4S&U$1{dpK8KeAR_G+rs- zZ@)S9v8bp$A;FEn+iuS&0tEl7)VX@Xtbfz3oZwh?FW=S*BL+rgCM0Z&(GHKvQWDo* zYYL2w&Al`b{Wmif>uCMQun0Z^FirlC9V{FZoqDiU;@8^lF6U1en^aAVtAR3c9y|T3 z>fpX!UlW6h?Xvcns>vK^N_b7k0KH=szh3hF74xX!BUlj9;49q%!9qT_s zfnbj&tUCM&k$iCcf+5-=B9F=uN1Dk{2YEW1}_?`O!xc&9G#NIw-^E7fiysl}^O>&`Xf=k&g&vf%_j9gwv)NP!*0Nz z{l}6c&w=8W_q_m@IUfdl>Gs$}aacym7>HxPdPTdvOOjU=IajAB zI@mwf)t=SXno(n!J~lW=MKS%jj~R0~Jh%!tGh521XxmvNkF0L@1#7D(3=GNv_cm>< zilhQ4f`dY7+FFAHS!>&c457eLQ`2yAjI_1*`l1O_64i5G?&bOL&(84oL=+42D&;>G z)EAU9h%hTp&Y0(8d#+RLJ=@&e8JWpd6j&RdWk^E}4q(nC0hY`5pZz`iyr;P|_&+Xt zW1jVIV%0stq~aAmYgZv6xNP4FzcNGRtS|Au+*mvFUU^vEk)czQ|~>me~U?>LdW}K?74mIZUS1}0<*}Gq zP}k{n0Q1b21Ni~hI}<;Cd}zkJ!~6S{NVzAhtZD=$dO#jt$sZm*Vl@><8Jzrz@A>Rh zZelF8H3z#6lfA06-?Y<^w#4$Nh8+n62$FlFHr9;*T7<&!eieT*z0Mc^xNWGgGHov6 zDly@Y+n9rY-yZCB51MRHWkBx{A8mKLx|BNX3Y(a?68JW)t+^_l><8Mc9jBAeHD6V) zdGbpYpt7qYwz7Tggs6)~cJRi<=7uV&!T_!?OmGue(T&$x>w2k)Ir8(;-Z!NUH`Il> zUC|K{YWK3ST^E~fWgFHL)uJ~8duUx?8y;AQymRH1SL4-8V2R)Q-H>vt_K;daeZ(Ps=ajS%cQIG`WQm)bb4j@ z>(TytJdg5T+r;4UZvMJH_ecnp?-Z~4e!LsU<3C>)J@`&mPcG{H(Oe5Q&(`i!5@raVd9IuU6&{Hze_sEe{zLe|-ph_z!~^e^xXUpTrLF1L0IU@f6!eXh25iIW z=~wi3lhTn75hFZftgBKyON)DEC!quE__$J>e0;U>K!r9Gzo?pGol$tT*jB%P*1eD0 zI*GM)<)l$bg=Jf*`o6UX5|!hB?M#E;Wn>+rlZ3ej#>2d)C-B4`egIoe$i_ zhETm>H5q+KiT|Dx|7GC)xbSdUU{!qnJWy(?E5O z0Bn4Hi3StF4;!r1^VMs|2L^0_YYN0uelJ1tSK{JuA9`zdw~d+E?=~71oqV-w7yv+y z5s&BPF&*bd(hrY}q^G9?`T@>{PF+5E+%>+;|Hx5p|y$hT^h5IqMu`HeI@7D$E za=?fR3SKxxWj@W!T?C>85Dmfi1-P4C(3jG^Gd%&*-2YWtT3$Xr%QHJ$>w}#GHdb+D zpEw+SUCS)QjG4S3z%-9FeF1!aF4QQYn{g%yjam?nFqsm|sTEbbZ{A2j_& z-9R69^PL#P{>)T;N=Mhx(gI+Z3(L$4dosem0O|s^@8U@Aa{Fx^n~EDX69NnoM@L76 zrvTL5r0Ii8wu1RG0=|&=Jua@w)>aW47T28xNd%$?u#!M1aoHcU<(}74PwMRKbaHkE z_)6y`Xif}!8@#34p8~(F8ra0r(!JoWZS zft3e+^~aAN;213tPy&z(ph_z#D_dSp1?PMPl%*{4GJra185rjHM&Qh~nSl8mB}y&5 zk`df7Ys5=IK>;9(d~kGR=`n?USt~Lfq^++W^78UBF@@wJ&?oo%BKrpD85oe<_Fom~ z0j0E2VZCe@9qGVcywcZ~x!WdfUC;Mp?!skLc7vyxadLLNyDO}Ha|zenfr;$r?AW=t z1S}<_Q&s@`WaxX|G(~4T%_V@1+LhiKEXDi&vdYx|51einP@hTL6ji8P0+S;A7O5fc9S8>1?2nYxeMTmw?5N6kK$dP0~ z%decmfW7Ov3SH$QK=mA1AK5<-wE(_Ul$kd{_5fcn_t4cI3I z2VdW1z98;#o0!-#EAA0DH#dxbU^BL(-fh!{zaF;sM#IvMvAb#EbS46Q3&%jzzz_*& z9)Pmg+SuImZSrZz6cgo%RkA@kSuBM*lpz@%>xl!DL=DpWS>rivi480zu8&aS;(Mr@n|{Shs#KEx2E zT?D)_EGYE5-_KH0q8+C@LxnEDv6b8Ih(1DRD0r@P3CL0AyciHN#GhBSiy8 z$4YVw;2VjG;&SC)z`~-TrUvL3sG@yPTI9h%YW9+rx@I7nDdT|J(k5fkH#T@iaB4A1)%M703DH9zz<9_NIah*%_eG`!9E5JGe!9MVJ8SFbPQ*g6t}a) zG8490xn>Xh$f%21pa%hl1GpG)C=E@`;ag`62(y=2v$WIG^743P@#bIT%8Dm!d`iA4 zt(s00iQuWcTFjAg9)sQi&O~=-CoVQN5K4Uz75%fJ%v0`&a}GdvKL8lx&mTV~*VZu5 zc0H~snD&#sXeqa2H4OTJxu_fgh-YPGx)hajy2QEEXD7GQO?pis1}7&^6Ps4Oz}^In zytOqY>UQ^trkEp4{U(uHa5_!4RIGIA>Tdlut=Y97Hitw^soOocf!4U^ZpY4@^a2YE z4;QznpnwDRDzHsBVE{hwf7$pE<&azxu zspsV7bpbT1dk|=I;0tjAwT*-0DhX#8O?+P@IJ^RdJ|XBb+$pJ1ORa5an7wU?kd^HR z7MHt>S9Z3cwsvYWAvlQW=<33XWKdz%Titgy1_QNo{i^W;WnBR@i7->pE`{=0W5&Vf zKQvN6PVmBVbCVAdw^%2Pl$37u_(NQV+)udaf>}*{iM@~Jr9CwsC9`5#KY$*lmbbRD z(l^28LDov?5VK)veg>jUodRA@B$03RTefej@Y<93V|bC{K- zMIh4D%HDCgO|6n6o&x5f#9g%ZT)LHkJch6ZEnMgdx9Ste0MZmd%bJOoc(i!Lu*TnZi&3r!9z-m7S?fPoHMy=Yu&O zhI(Qx!b3%+y)sk)UNJxbS1dClJDS}Dl*ME9no&5{y1Ewy_QBBzgHGu2FgcV#PzM}M zEZg~5`q}yUo*1?sK%xT)0p_&|7G;RE08a6cX@&n?k2i(Uh5omnY5*k$0*{rowRY`h zo5uE?8#iFTzqwmqRp@0B260OQPM2dl35g(YZ||uN%Owd-DP{1n`vFiD`kQYkL%^;L zNjcdtvv*ApaS*bw1UVc4Z!kq_5r~AVtE=FYn_=9Y*_CLFkV{>_Z3!UhRS@Obrgf_I z{I)3nJ4}4l2k0J5QJozfbPNn|)9IVTK=$|PFMEy@05y+l^dUg0^Qyn$3=wl_VddDhza^*Ig%xRts+d0?cpq? zmZ@niWY0c6K3-nfXt58NFdxfm94$(notepDw~>&ZPJ?EL_SO2G8d!sko9u79q?QD*TpIaZGZqy)eGM!wG|7kx*CCiDq zO65q*sAZ(5_p`imZfO=jEQQ!@fm5sIO7Yvq(2@^pJXmeb#30*Mn$Gsxdj4Dji#uRpXzxjv(eBO)7q1g zswx%g(8ZF$DrLVi02l*Ds`eCp1Q17-yYNzmt`8>XHx8hnbmn=Wq_i|4KK}PYX_k4CRww}V>TjSmOPCuD&>zP>>A1z=pVo!75v{6%^Pt8nK-XlDYC& z4<-V##sfn@-~CVN9|0QI9yvmB&Px5IDh-{vxjCdQ0Ey8k00O;TS2T+e`_XSaQ`2>b zScX;Pbb?M!$Ezcy07RZ!TEZ|EmzPIYqq|yK<{?SA8;@{-^s2!V55`SlW!Rhf^nixK zSJn^w?(+b13X^t$Wo6I&H(JOdR@~xowHLl8t~}lYeQbgkwq}6KGGf+TS!a0vN$Pp- z+1(jpugns6{NG6Uz9QC%pXq&vj1&y%z!wM$kWrzWWQ{#N`9(!V zAk)oBe)aY;^YWr0wt}1l5ZpB!95Q#no6gM4zz~IG0t*WZ=_IAOx71HZO${md%+6x3 z4~>vyAq-?#Kv$EM4F{N$ld2d)1Uxk!A`Q$=u{8A=0JxC?aMYB%fyWmM3kzzo=tkEP zAZ~+Nf`s$r1gKb9tq4IuL3{i1t@9Uma_n91l1gUh@eX_5fZuM!$KMV86|JVG1_-ts z5H~g}Lo36D_K*RTsW8Yt=3=q7-pzCb8?JmuF9&J>u++~fgSD!$otNTlH7SriyYoiVh zA|oT;I$z@X{vY$YOfH}P|I>+`88Wn<12T9)!A8h6ARGZx8nPk#YryBxl#vMo({U@S z9RQ1mC4++3{{8I(V54IBGaoT9$e~rj)A?a+!oXxHfdT_FGjnL0mJbPwMR{zOu9jBH z)DJy}0dBqzh_k;bZ-9O7ojYv1B}D1HE>r=c@ZY z>+M4JHO^3MGtU5s9puAEAF70PNl?gTStM#e*bF7C4xp%~iH z+#HC&6q^xDgBCdAo-XyhgMq7z!iKLNdV76EGDUdY2i@%RWPLp&`4I!(B)pzLroxc}V1X zXf4e)g8r^!9FVE@_PRSdrcYY~TuZKx^A(eIV<9wxx)hfSrH_ z&)5I|ncV+ZGdXOhPR52Ov64l0n^W%Y?kOoLvmFuJ_sVQme*tna@E*s{{(jl{zF`k* zh^Grbl5xl&Ycm61V&I!uSls2bdWhi7d}=yc_SEe6b21|E@NRSwYX>nuKZu0luw?Q} zoJ@RqdEkP-_NT$UdbO*&8z8e}y7qP5^(%cju3g!cyt(Eu1o$D+NcGXUF8!$}(a~MG zdX(3mV$;v|gR122MIjc6-3_R(`MI#Q}8TaY-SPGWZ2v!#$o z(Q2~>n@de2LP(9JP=ql~l*>i&#lAcUuZ>XL)dBepNXHK!K7b}^;lJw8>JW`@vG_9; zpp}e_j8URD@88ce?@v0u+%gUMGvt+$$4irwKOn#ZqO+*z84V4MP1GB04Gk%E^$94S zgH!Z!kOPRXp#n{quOR;~1g!R6$F%1)64-rK4(ICT=BBYt&H~xBqM{#;eEMtXXB zcGFAjitRl;@oc6@Z|^ps)LprJ8Eyv$9A>6;`sK_Z_#cx=9P1K4)o6tLTP{xpa`<+a z>CmP*=7$d#pd~M5+S3qms>IGwLSDBC^lK={Hbu<$B{rv&=m4@33eyVokW}U7=Kh^| zOUKt^MFa(Ed(a3zPAiG~3~l7w_5yRgGIDdB*UIN%GdJXBSq+5Y9;MWXE<+GBRtFpJ zVA;aU%PTDG1`SO*P&M^frhql~(ajCAzVqkLTenupb(upu0;Rh@TmB6tfq$?^C+i`c z+PY7ycl2Emp5xYF3aLyhC|QDx2WisBI^Q?eHtoAt#s+_`O4ChA9Vb zY?wuf0;B>cp|*=ZsZNCNJ&$NEJTj>JcyYcvPP(QROVYCT=VEVqM0hyuImnE7G@Ehv zbk)_>_ui28ZNfI}Sb&5=t`BLPV=Q)G9|vFU;ZimfQ)OjkflCNsbeR;rfdw~J!TwOF zsX;se6{azeqsWa)X};mxgt=Dw`ioJ_y6m>|zqRzgy*~@(Yp6Ry61X@~TWi96alWK- z=}`(4iBeNj@2>Mz8xAs*!0y+c@p8*_az23iVGg;-K_N_FA~(7NgZN^CGk2 zWh8oDVXoT2Oei8LIoUqzW3BR*=(((5b-PKAogFI@YD9qoR3~(ejH>i|sqygeW+^WO zutSs^mw^Q;zcGsLQ2YMaE9=(nmTApwx0iH?$xcFbt1LCLV)$ff3=H5#c#z#t5QU2K z?C}*$0XHFOPsDUC#5-is#0FhUk5@L&A(=Fc^9y& zay?CizU}(mSAWxfz7BE_kOU7Nd>b7-Hc!WNwyL|V2SS(0a&mJyCSsg?G79~_%~Raf zs^YU&&Ei06@RIh1nQXwJH{3rfW?WdX0g1J@S~~GLBBDc!X>2dr*7tg;LH{F5%kBLM z*HeI;zD(w`9!ur;ZhYo`yiP)|<=^)LsAZfxv+xw8z`1kh9Gsp)1_HVYZZ8ocDe-P# z9=W%``gb;m`JTYn-!jcp<6a@eYyAaKo9^xx(Ck1-;tW zeg6TgcH>2pH!!Tl`S|$u<}om9#2wkr2hc&qUcI`*ZTAIT7y>JHDk>_FUqK-b$>Vqf zPtwuJ*TQX(Z=Utt($L5VXdS&O^>IQ+p6?>9{-5o->ACU7tawh?zmBPPXKp`#A_4%4 zE5d%a6A{K%E&Z2HMzffd!bgS0Kll53S3cg?K$XM0_ z`Z@zU`_Ryk0??MRu<{iPKV8)&J)T@$bw(fC`^#&2cm)R5Iv?#7>G!IrsscsW+1a^L zwGcG=Jx(i;>Fu_BC~O>TOu`VdiLh8a?lZSz&AjFgd|_Q}ZE!bCOHExw-DB65$_#Vf z2u;L=Dr9w2jVN{R>W81I{)sxo-F9qtUFXxR-lta~S)V@L+}s2o+9`X-tTK~CCrwmR zQc^}n284bRVRo85vHP(oP2T~NBuaJ;|_q{8L5=~kIoaa7#;8_X1MnHRJbQ@kq74w zV6Qki&wT{62Q2iT7`A~y{if+T3bNm69rx1UF%{xR!{EzY*!D|HbE9@~bC*7QmLHaI6OiWW>GD z4ct@Kwg)YV*ZS%Lu*e_2RH9u(Nt+)^zJGsQYG~|3CbniisWm~jZnb0hC@(Kp z;xH<4+}+d%bUY{^090@A?b|(1%&$8ZU}f?Srv2|aJ7aZt^{|!vbyDVx?k&8V4i33B z-|@VGmJ*H8$i1&D7?r<-SwmNrQL`y_m}hFwhWHyGpmC6PED!ch9t>8y&kC;D85Y~Uv=Ib)l4u>lGWq+L4DyD8xh`V<2$U=WOwN2S^qV^slB+?Hb;<;um2Bj z$O1iw<4)t?r!M0ku?!o>>0^_UJC!^-gu9i?b4KVQ9%U7a_c&DbyM)DWyMkFI3=<)@ zZM2EwS`n|5T%B4&IdPdSDoxCa4NVdAn*U(X38>d=Og48)qf#-AfQF0PG!He# z;lyOvx$}X;O`Zd;c^K)?uz>0NlAfr2ioBK77K82g^|7&Y!Bs9`W-!SaXg2fK$mj>rFB zNzVCRewbJKJ40H*O2Wv1krYH1@P;+;b{{WQqwMcFJ+olB=6~2C2Yfyx5TM?Fuy#y0 z?R&YBUNmsheT^nMbgU1oMKdU7>;y+q0@9D*{GqoAnR-82ZY?c+O;v}ZZ%@!%9kdwT<=$C;Ud{{H6akvO}m ztCf6w)Z=ifQBT8C^qC~_iJSU@U@kuZ zSP<90g|#_I)2^YO!2QS0U>pp|j%;yF%gY|Gb{~*CZ(6u#r*n^Wx=E6BH9A#%5LH!G zGqZcpci7qrtZX`P;k3&qWddLQ;Y0dZpX|fJ*V}0bgZ88Gf=#m-XNr>vXBw7guA@oE zYYs@5BTRK#t+>?7QJH{qXaPrD$E|?csr%C>1*C(4Fa=RfBjRhBsH2hXLFoQ)SYzk4 zZ^V($w)a19A;+lo!t&lddmNmdy#|6MiP&+|fTG3Q9J9dPxHes=vV!11>>T^isiUVSajun@eP{pJn6Z&j#`KNkfh!qCM{gtjMkIrPmJ2*6ksyvs}kSw$^X z!NKFKDP(lO<`E2|0gIj1rkO+0;KPmxl@Cr#2&SF|56YJ~E%=20QN_K7TOYCXynpWi z5)5O*eU*^MhPnOVuO7Y8^llwutMmysb=P~{5}rY4hvMP(tWHPc@=V5F_LiLIdlEW# zqP#Am)?*`OZ%qxiu<%7(d&7qZ$&H|66g*i8kj zfKtDS5`s+S;>C-{Y)w;#fHv{-j}L?>Otj@B`TA}dZrs3gWNB$>cBm0NT8x5^H#mE{ z#=CKGGA?7f>^ntW##%8{gn^(}4x-0x19%8Vyu;oruib0wmMxgByV)crQx$Ipw#456 zYsx}HV~H_taVB1xFoWZFbarr{419&XL4}2p>HaA3p{?#?I*XsFW^%KpZajR|TGZk= zlW{>r>gWB!HmN4Lg>G~`0E}GRQu3xg4raIqW0WOhmH#?COh-u>cIQq%CcMwjH)boF z;Bmir@d;o|yo+QpZI7e~*O8XC$YcJTzfy`w9>h6Gfh|8yhKc>;d03?!mG~lImGA25 zSwm~mITYW?)cGB+$lHrw+eLz3wTdO7gcHU8n!)K=SOmlCv8*1o58pU-{nl?I&2E=Y z?fq{tbJE_(oz^in_Q0}vOmrdCS7W7zhan`9fZ_=YCV_E-d0wqzT{}R8-S6u&Z-hY1 z4WkQdh?NK=klUwoF-|K8?%gZrG_+%Qf+=2rlwOz(Jx*zgzJ)+nD}?Z_GpK1CGCtj)3Cwsz1oMJI`iSgV^SG}w=BtP zM$Oq-TuX1t^+=-H0f`wBp|VhL1yzH4Xai?wW`cr)B`BA-*$E`{qZyLZKb5o2UQtO9`2lJ*?X2;AS27 z+ozu{oOq@tb*xwLd{Ei5j($B5;qPnB#Gp88BD|8T#E>qPhH z@Nn&#EI`@sd+rA?4C0{#^?iLC`9~l9yxd&>z(CN_j~_ki%s&&-&wdVuH@m6j;cjjX zW>3!*%-X^5?VZlq#<;!YJylbT?{IS5<8)S)qGxZUTv&*<)@zuX_uZo8oXZHFc-r~d zN9K`-g8G>4{cp=5Vm-cXUQs8{GG5y3(ou3`QEbrNE`;daWd@gotm=QG%=~&?p{%^T zDM=w`Iv|Szb5$|YyTK+y;bCi=m6Ky%e10D*YZ6M2jhFmV;rTWk7_v@74HMde2}#mq zP23-@3d2}~js-X#a51^0-_kYsBJ;?-MI#``%qwn>s<$kaJ64)0(rR19Fu66y{dol0 zR76_~^bQcK0Sc3~*V@^(H^N@vl&&t#&YfVxDr`GM6K6pO;9)9X8j7o^s1S|$+IGrE zZ*3gK4^q%N`Eq+mFCl7ql$E8SsmUyUmO^Xn1_jV5fIa>Fr(e08g=h~l)A6<(umcJ5 zm!-M5Zcv~*xDy_ZH#9FlpP7ltZALA)19}56NocshiSQ@V9B#Y=eXNIv2jC$OkJ1?z zi4qeQCMK``YJaEUw+@L!_s=z_s(}^@IF+&UCC0fJaT}`jjv~8HW0R9wMAHiky`?Uh z914JSpm2CiPV+8*XvxM)DZ6ypl2w}mV2_VCfy?o5cW1^}Us}2h$PiG`5CQCQEHU}B z+Zp3CVoPLsd2laxVGD5+k5=*VRn>#@@dSz9W=L6p$Dum|KKgx{SN7y_m+Fl3npg!x z<-g;Zh&9-jl9CeKwQ~1a?iFW;KF*_jk=P&|5fvqLvQ#&0w~JWxIT7ClXOwoE*U-*W?p%0Boo1hn~B3IZiBg4;e>s3FLfGGEWo)mU;+qxJcSSPAx^Uk+> z5&K*oPV$x@z^l=bkzWP|EKZ-kHqS730MGmFfBg`Yc?JleO{4>TmF9k_?V=lnz6eQGxOOk?#i)t2Uti8Mq@;t z#E)+#x-&VLd}wjip~9&`ZdljAKtcEUs*xaVgkGWRxAOBzV@{8>q-113+&y`#W%>!5?p`SW64j{`(* z9pkmdKaQSypgr2(pQ6hb!z|%Zp7>ErNGL>#u~Gfa0K4p#E4$B5_LO6QD3F_8kqeaw z>%pHnQR=)Wq!ySl*u|Qs^c_r6mr|WmN_Xl^EH52q5C~^X1KfC#EXmbrW%Wj^Y9!g84&UC zLx-$)Xhkal(&XXu`D8{0GEVP<>!+0&=*6ri%z zINFA`72I9`|HI78akPS2hlP`xCck}qmA-S?h%cnJWz@N$b`Iq~2Y@Te?sancXo>T* zs}3<0q@VlxUPy!W<;k;lauPctY0>!MT%jsgih}IJ59g}R4HlnpmQE-ymz!bO^oEU$ zj7%V(v;OT{v;&w0GLz*C60ya`G|JH4U86nJKEJA++9&M5Mz*cF{EN-RCmi=>buE0l63nvcQuC4 zBVO`8Z{wxVx>{IAC9}l>^uzf_MTKA1OxtPMXbdlP^y^-)*~Z(i;GX|$p|XKm>2>ATBT^*1cYXY_Fm-O;nF)V`Y&Aa`(AL5yEzWL0e&%5(oI)C zLspwEv_X|_7s;-P_ORyQj_@v~-INT4G)GsZ(fyx=>^Wuits;S&8p-~d5#)6HQxS-y^q-BHKlVWGR~4`4H?dP+~dkE?n5dx5oMjD(dTXFplX zwR;k+E!WH$uQ_jXJw7EDFo%R_L!J5IIg01DIKA89kyx9CQUIa|BfX(x*yK$Ar z2L`HIB@Nv3yK_C}$AH{IXn!Y}(s~zH^wLrKfNm1v;?OQH3_Qun?BE_AKeoef0GUPi z7jBkRnTI}J;XN1i0{krpgLg^OM~4JPn;YLxF9}UO^^$PX0H?+bX!Sm)<9%zG*z#zZmD)Y{rwQ==SbC?9csi)6adHAtnDG^iz_qNgNCh9LyCcVH*@~BsIR9I7Ak{<>#Sd5+X|J*V)XaWGiaZ&keaLYhM3}W}ek*J1Phxz&W(a}+W4|wBfdF}7Y;QHaifq?!p zF=3Y{s#cUcG$bxhW8}|hbjud9Er~EyU44CS+CZQ~wvUHYU88r^n?#!d*AbS|;GgFL z<{9n1T}N7!1~orY-3o&&I@!;k@6C&mZAcT3{$v1z38mj}a^YT3S&le6JGR#a?h)YU zKT+->vFfR2r(S2#MdWSvf7Krl zK%A}5A6`Edw~6ounG~Lj)Xr!fb%t=A%C+Q7)J?d?j^L2M+Q&|Ecd?_Nc=_d#23&)i!m%MlICkrJ?20VhJ%gb!zhS=ZmoG zjEtN;M0rE(C>PaxA-%fSuTvIoSwsw2WxR;#f9q)P_X>K8hT(=s?8b9M( zpDw?)!Cj`hf0mN9BVohKHV?yno1Z3$3}jpGr>s7mtNuX!=us58%k7R(yOJ7I?WX>t-td`nZ~k+l=LG}k@TY0U$pf;xxNLEz^yzOew{BozxorlAp7jpt1VJe&_Zo61SWyZK z=OG!QwWZ#cXy|&N{f}wyFMt17CC4(^w>_5D!}vLXFLI{+S8|+ybOct0EV}Krr%tBt zl|$B1#x0LBO1tv2;nXjM#1P*#P)%MjDkQ1Xi$}u|}QL6wuY~HfPC@J!WB1E(1Cr+FI zo{AzDeW;bCC#8Gn`F~%P>wg?`5+WFjv=}ScjRm_Eklk2%dG9_Qq!Q2&zoqatdI3qSFlB2Tu%4pzf z>v8b#!g6w-Uc9(kI{;G|y2m)dbn-Ow_YZX>nmjF&n7EQ6rR(AoDKG?B#4Q= zb>EGSHavOq!i?7p$dr$*t!3Xk&$n2a&|>D3jk(D4^4b;>NFrC3m*I(X;%~Ytri2G^ zn?gqbe&F)X74-+rdA~hKXDoWXFKfxp>CJDU%^N?9rq)(hMq5_k+rX>X($;2R zY|Kbce=F?ham28^KPwLHWy9r>!-2FgxGd!Otd0Z3k%7b2zDMJD#It8BfL05PFF+SgQ96L3c$YFHtIqofwSx=Bu(Q|;{tyN1N>p_(8KB5p-PwqNIghnb}!9-AB~bN1W9l-^ghruvu#mOWSAd3 zWM-ZyI2(qY%@EFXn9UIG7`lPJv2c|z#GO~SHB`_9zdghAu?SILr zwJBkI@eeJ)f5xu=j9vd3yZ$qF{b%g@&)D^!vFkr$*MG*Y|Le!Dp{kD?D^8KgQo7&$ z@#SpvQ6y;W4O1QIg&0OFN%Xa4XdJ}U`yDuVdFz(5xM(1HBZNWl%j-#I{RG1lGsDs<@&`-sW)yA_?mv352A12 zCxDn`S%l=|E(5!BmbyHp0=YiaF&aH#yUj6>^S`#b z!iU3q%RP(19T&El#t9DOEw(y*uD>BeNh@GT$T?_hTUZ)SI^j+h&ko31ez9I3^z8eb zb_2f1yUe_DAD-wHnK!z|onW~VLAC+ES>y05^mY9F{31!rF)OrwLU#=^bRkn9_U+Fu z?s++I703Ku{L7`BTuASXkk^L~TP6izy8Mv(FO) z*CLcVyaC4WCQJ|1GF-M4HL|zQ>ASi~-m zNIIu-O_~74fM1aTS7PyfaG#{LrJ^w9%S}=mmhbM2`)}Vh!MVjQN!68&NAcptDNK)K z%+fRr3~H<)*R~&?2of98EH|}3bLLD}Z6jb@sGvwmNOmzWBx}*}Bsp+|jQ~!9hOlz= zn>|Bhkd^R+aQ*$;=70lEc}@Gq#qIJw#?|2`yPa*Xy?_tZ%M0F!e}c?CZBt0CPKGX6 z(ZOkd=WSG$T)Vnn9o%-kuTMVu&EpI6T$L3AEjjfBj1y!l$Upfc@^LYR zgizpUc!g5Q)2Cd!cYkPYedi1E5(-K?=s?UHq2B!lbw@{s6es6(m!rg)7@=b{E{(PB z$u;~sF_9P-7id!f^*gv7T$B=%3j1F9s&kO1k`NJ1&COYo1{^tYBCgL#FX$j}5d5Rs zI$xwZ9Ua{R?fS0|-;d81B!bmycz+f*kGyZs27!Yh{~F)Cx&O_-ck*hM|D5%S6J0QI zyUUs$J9f;$LF8{(+7k5zefE-p&?=>^drK1a{eJQAKcxb4flR-*j=HY#rbo+~L?(Q*6jG|r9g``!3UXphewr_EW0}=Az;W-Uk1N>H6QpbRmXQhqZaugdlh3;UPT&kbC?6A2(UO@A*he$5JdH4P|=i z2VDLQagrf1C;hISMIHzuj8s)`!KtIxIC>@nE=w~$pV>x9nM+^qwWxb|ECYi9fH?f^ z+o$&Svv}0)?VA|)kXUMlsfvq>E1gz@<^U{LK#H=m^5#uf`+Q=$$uI2Zy?OxgbJopl zEG#TYhT@hfgEh$78v0HdQW6_E6&0136Ib+&6UWuJ1{QZPbhcarTm%FS3hZ<+MVqg| z%#c*Ef$gNNfbCNY6?I8#Ejy_A%^C_8CfIX62mo#LY5NZ<1HHr5lS!+fy3O-!?UgB z$-{?dA-=*H8F=f~6GQ3pCp|zO0We6K!2C=U6VnMe2{1x?Eg`69tIkq(>&^Q&6h)e>88^)bRc_?Cn$`QC%DSlAtB^Cy(V88 z9c!UCNBk5nzM7wM^o3BfAsKUCMkGO3R(cq6lNnKc6>=YH}7CLcnZCcZ# zN^cMMoiIte5i>J+!P9;=!HGkPcYd)b;OiiH_keFD`4~aLa`*055(zWEtn@trj7&^0 zmj(IzALok%C(CWH*Uw=84v>Wq0y52>0d5`@)rR{@$TWj&+ao6MM$<5~;L=i3R@TBmZ@vATtu0hNUuKsYqK1PA1z-qoUZVRD153y-LUvlT6wFP6w(p`ErT4s} zC3vScZ}3`ZmAK?tBac`a4O&9oeu*EF=rNW7YY4KPNRH}qFL2k)aBn9i&CbXOQ{{r7 z2%iwVaFpOl(epJQ+S(AkhLV6c&hTED(F8pcQ^V`mP)tT4Os4)dS4T~3T+8Ay%iRPj zg&>4N6fYo9LxBpfrM!|@&QIoc;C-**8@?D6&Tb9&s@!J2yjx?m?pVXyxB7gMA|fJa z2XV4IQUA?)|9_Y1iZK1(%XEG1Z1DK@SmZkFFE-+g;gd=7kVMLbLTa@N1#gf%sJPNjTTTMtexGufo{lq) z6S(&60-hS@tlaQZBqFyhR{fc{MbFmHi|^Bz2q(7u*<8Ksvaj45^+qJ^`OV$j+b{A% z18^+)m|>_a3jO8F<=3xQ!4(1imGm&h+x=mxN|n0cg9CdwZy!(0lyYxn7}5x56%Ml6 zb!V}vU6dGfc&k-uA)4xb*ek{a(;UftJu3D$GdL468!TFDJ>GNw*6;_*rt{iCr%?vA0AwSFzhJ)N z?dq9jAG5rRuaD|Ip1DyKFTLlB`S_I`Pe!-!8#P{`-1xl8m{@FGV4`}eX^6FG*IHF; zd>+44^09~dnXgWZB$EkMES@$pVyK{Ki64v=&u(iWKlE+1naBN#+>XETXtcFHOjfHo zF!X1BP)rU@rJmtBV7IjA*}H;oJ7V)nNS01Z((=uTDoPF84YY90#w3Kn-vd^$HajfRa)vHiT|0GJA0iMf2F;uUtEqytjWsJJ5KsZ{u%vrpO~ zOD8P7B2ZcseBb*6g|um025Lwxd>itQR^3cQ#3HoW#l;4{h9_rNR|UtxTl$&Vj~>k| zEYt>lf>a-^{kg)@`}Zx7WEf2tq|RHW&cSm9w=KbQhdQ)%>*-j|+M{IJ6jX!4uTJMG zCXY-`8t~FItnMVR-PB7#iCsM0G)5a~p{*RO3xihiu(I;l{4xw)79v`2a8c}G&l?>@ zPgU!tlhcOjqvMiA1Ev6&HK2Gwiv=x=3dx;%ON?5WR&bH&TOdz|qZ`QyYu}42t#lRn zLYV$Ze2qbE4-e(XXkVFLqq{1898%pw$ZaGgCqKhCAz=iND0>F{{A}=5xP1$hv*>v z$J6k0EJ8$=5?xVY;og&_5w!!z_JTIgVp(b58EJN(XgDU7(~Z5@`PD#oN(u=z!E7fO zHL&2xc76gC3pfKb8&FcfMO++C^P}7Ih5s4(>{DP=u+0LGju59Pn_%z6ga8 z{;i6V=`ETYS^iAekGKLAgv}R%TJdq>U~|~z(~4m`%_d=>hK@7li7sKHn)g^mm>TS} zvT|}Z*`Ynv^c|w;dghqUQWk_koDQY@b&->>#F+>p{ScnQS1O{8(SyfTToZp%yX9qN z8f2QhH29206iI`#%nSj8*Pb?haV*j0v*HstRT1wE9VX%Mexda5XDtnDHgaLwI{zyr zs_Y93;TjBNI})iNl)EchH>ApGF_+34&@_yPvH!AVqZRoc zA|q-kjYD4`N3`;|qpI@VfBRwmD<>Pcix=kTkdaejuJlfs)?aCkXQM?L00Rxp-^H4st>xwMd-Y8DNkXb~^Xk=& zfdehwOK)#*l(5;&LiGcQ3QozE(aoD)zz+$y%wFG%nX)Njqdps;0bogBVMSRsX{JFi$|r+S;38mpj!+thRD5B6Z_c?s?ryH>WO!7YL%^Z7_qu(P*M zPl_=Ayzh+V_RX89vITmqhet=HL`4;k)2z01$HIGsP8}f6*w?R?MY$Zc5ICb%HoVLc zM|h4|cI`TWJeZbIv?KZXVm&G~M5Lq=MrSoB=mv#JBPE@AlmGPqL(YT#7cEVFeLWO* zMfv$v3S5E3AD3cwdY6-!G?d>EahyWE|fBT7Pi#gV4@*D0wV$L23Z;IDCx zHttpT($VM>Q;?au04&1kvL_%QBqYS&e+ecOGr|Qvn0)DUbfj*K{9cm7j&j?bZk3Ei zaXpBOWFY#jprKdF{yM8Pb`Oao{1-!`l68mqU-h0a2(TrEWFst7t=6JBq92~vaYietsLvnlR)7Ybr zS#D_`9OSyZZDcsfCow)=+<>A)GW%do+bP6MLgt4jkBcicGn1d4{i+f1yKxo|5z7YW z|8oAJrKEh{)^;YwNO(3u7)cU5B}wCkzTHEBCo$r|6Mz&GNdC^-+Ybv$PP2VFn4@AN zaO9DOFhMfPlqTU3>mw(cfElQ?B<3pD^B~R57Ib^=qz1H7;|aaYk>mGKgMqM?V^7Jp z^;c(#o@L%_sidnuMx!?y<1bWl;s;eZlr(RX6cjMEa4b|Te*Z%Yz|kL7`D6Dty9}e!42aMR$VFP*%)tqz?FfKFpNQr0Dp8Qo<1i&#s05?Wqc{ z4wFVeUy^r9gkRw2m^nu4k3r{gU3ir$&DlxpH__*Y9vnQIhB$mR9{oBg{v$?;2vF!D~UlY@qdRZ>P`ixK6+!*+MLX?jZ_? z4yZD|e|6A^`G}f)VVF&@?1Q+S0XH`K#S=vb(x_=49`i?sB41)~=zJOi__^(b6~02u zCx-_i%V75tHm~fPTV#8}*V9N^KB@a{gnmeGQA)g9V`pIZqmUYc7OdwZc>i;?JDAbVe8;pdzXku)v z6y;S*era<&{*s7u_PGikKJMol7SkK^iSfyW-+v*u zJ4@*+EeG?hSKZyB*H)XLoDVe050bd=>P+7?`Ql5Qj{Vx@{^3VOU3u%DmWx>(R5sjM z*#7(H(>X8q?d8C#+q(Sf3OPB;)BRFLLRWf5`$orf=Z0qMMa(bJ6fPNxwKMDBfA>LeWGuGS zQtiSi*2BjS-M|0Fd!E1IzC|;M&=W?Vg(~u)VTHBl_NyEeWxj8JLwkhl!P5FjzPvF_Ki41KB&z2B8xSOsb%vFVr0=z^pc75{Cfd{*#0Q)&E!H%r7;0^|N@4@)mPqrY z8e%H~)n<>>$($S^|ED;8!crar^J63VT|^i$OPHcQbabd0j?9KQ4UiuEJ~}XPy#S$? zbGP{T_^MBfh-pNsMVGkOE+D&K81LJ|D^(w79sEcDm8Pp{%b_W&kh@~LA?T{-3yOOwLd3lat4V~4zy(`oo7(cz3 zjBM$iub7eXvohhB&9mH|-;Y!n*d~zYZ+qrUdb0ClHjA_Y4}!r2pHCgN9v$(qV-)it zD=mm893LGu0?Fm%)0vff3PErpA`ZxI#bDk2K127>srBWy8tzTn?t@=H^M%AX{Ce6B z8;&c?thDfGB_gv4^6k1^1zz%%#l>_&Z7nU~DZDHp=`Eu>-jGgG{=yhat6^BV|Ne92 z8S}@^0k-P}ra5%}N;HT4l{jDr$SbK&XhG;dQZp2jkEOS4-+T@E4M@{SETfQ8wvY(> zF3xoP#ED%xBmMn~>rC|Y_eegRlQ;=IFdZv9YCB?QvlOd_&J{9p5?M%dVG=^;TvYTO zs{~-*urHfd2G@dQP7o`EH`!zL+{bobMRV9>Eb z4kF{;i6Wyn-Yv-B=lUupt1)X#!~<%=?};HIm6s3W8aKB!o??vboB~ScZhfV;(6Yv0 zM;eQ$jH_#-Ba1%QYjGtX+c>J^ArzaD@$^o{eRM>d=?~Jkn5TrTeG&3mdIR~u+HzZ^ z_tKXrf|{W6$D2w5{~_}o3nU0$_@?zAO+X*3Bfs?ZO+!)35J#l?Wjg;i)~;W7F>ZdY zy81?Vcneh1_z2SyEzQkP^!C8yg?`cnIT+ZRfHqNxk8k^6eYq6~4kv)oJo(OQln+h$o&neSLlK&e-PA+0})M zvL}E;NU4;{s3lr1h7yCGxu4j+eada|66f!#)xt;a1;e+WFGw?uSui~$8nwIk?bGK7 zQ&Lt2u>%{`4l*+EHj59UOqX-kmxTgSV6ZXu{8pHj#!XVQHo3kwS#yKHa3DrVd>>`# zYtA#<4BYE<2p?NGQl#8x8O{Ote-Tn~^vDr0BX_Xf5U_J|D`)5;*j|U`)$lBSOC02I zpI&%?0QEoH`*j3x2}>N3GGwzunk(oGZW6>VLJ$3m%9uXy0iy&`_cghw5R3wM2#k_( zCvHXD&a!DpY`BFxDg52$PnN@AN3dEBB9M4_r{?EfoSfd&*C&%;jIcTQi$-xfMbQ%v zQ0Gq~Va{U?cRmnKW!MDE6VZ02PaZfSb%~LjJOt()>~+Y;wx2&wxyIFZKQ}`?xA1du zH!ZR?0tc^rA2bN9NeR+NGT)!Y{fII6ey8r0eJ8vee>2Tkgcoj6J`KE0ON+eXSWX)p zayEs=A9t}oEf1@7GG?QIuM+N!VCAZ>?Ce5H>fqEm0oVa-I#+01C3%wu2fRR-@%z z;SlKOv4k|F+(i07bb)+ zh+dj;9d4A}aZ_!C?8lo$jce{3OJ4OjU%coxoZw1Wz91t54=>IQ))m?2@p)uFYuh&JFjeP{Es(JA7rUoCUPpf3I!Ptgz3y&tv;VoOYV$Tv`-w_P_ z!on22$2h`Zk3s^|;-cH3S7pxNP2 zLizVo;W`}~C%dlIkKm7?+GZ!!y}XDP{Bg}Ma!k~+#YGQUSm2{77`Aq`D_<}>c ze_aaOCyDazGnN-FWb5)B?zsmR7Dd7BAPnm?ho{|Sfrc8sm3|Rt#7}^f#2J4y-?-SD zE;2gEFAR$Ac!VmKc7`r@TwJ#l^Uj@m$B(nq5W7ex9AMk;wKxT9R6R)}W14YxTI9m~ zyk6!9Zqk=2B_&e(_eXap4$A1@IQ!{U{)QlrV0UWj9d!>m=eYaY?LU=TVe{!a@Is&Q zk9+fD!;jRGhj*O-+F~MjW{oRJv}SqGxW}-at*vG8;lP+#%s?F{LwHuuD;|U0`d)-r zTT9^fzDWlpsXIE}Swfl77mT`j;|3z?qisEr8XAy-Ozr=wke|Pg4|oGvZpUaOkUBs^ ztT|@pI1JDcO@JwK;tn52bOsu7gu|4%PO_LM8&x5VFf|hC-IPR|hZy0GLRT5q)smHv zA+XNK7;bum{rZcnhxy#}h>2FgGX+2*F@<-CZrsAZD_(ZU1`qP;O@$vhHfDP(Cu4a10NeN17rC143{HO7qyr z$Rgt69-pc-)zzJQ*1JlGld7-J$jp4htqw^A(2BWQRv&Sra8Io_c{$qXARUC_vlJ26 z-`8gj=l^nl@W%g->dkcqZex9Kcq>t*X`gbkv3*AFCU=Id$L2MUsu6_kQBOb=7Rpv-US|-DOfpD8x$y z!!abN(}6oJh0RjzV6QD1PULhD*^b-OiOd0P27^-z6$AIeGIK-z{vigCctBZg^GaS> zsQ8h(m!AEjQ}F9Gf3^R%l2$%POk7szw7;&utbBbsJFbFoxY6J0`c4G+{;KPew(VE! z?QHMwD~<%Bqr!T*70yPG99UEqXHqOoB_=A0xRWVgfVe_JTA3dZ0rcdOXFlVde_Pzi zoIvehj~^>NJ@(SYBilml`jfT6X*WD`jKOL7`QhFh19OM;D~xw#Y!#&!J{}bs3P{bW zIqnAK7bG6Q$b}OMc=K$6Yfp0V=I9l5CaTfs=vu>3z)JM|<}c$crEcxmWm$UQwyFTF zfv)b^LSvlVG^d$r3#k@7DTkCr>&3a6%xVlV^(ZD^Ir0-Ni7sxfy5f^v+c319fqTQW zjVkU7!jq!uaA`K!7X4bJ{6DSpv4f!{qdRx#^V!dQZr~ACe9lJrZf3Ol-;xrq(X~Rw zTQOa;i@R4$^Ze0=>s2SN->H-#cT?4WSr*xCLbvOY61`bNerU8bhQ7Q>?#@=GZG>M<$5rmaL>~>#>7^eK5hL0=AmyK?Nqb(X z)fhEl*7NczOMZuQ3j4qk6Iqs@QvCbbpVP}&lOtjD^%E@%Uu0R~VJ$1`1`MRcwAMO} z?io=8Vqaugv>s?9dYp=Od2x|G{K$B{w5#6(u4`_qfMf>XVm_~WEi%-g1iU9$Hie$kgJ%XyNu9^ zeUg%dDDftrwR|j6;4fk=Ke!`&L$kW-s2e|Ft|uT|7N6zm$cTv%ux)?#tlO7JgOQbU zN#4sgYHIZapC}^q$mJI|^qQNQeMUIl9KI{ErWx)kU}(U=Sc45sh=jHZNWsXm520TG zhM|Ij5JT+syXDY${X3ZD)ufgCs6D1WGT?OQK_(l*zq zbzu@!tUL5RIxv_}+w86|@4xst(!kW_?Af}eCURop(-N*uPH&Q1mmxlHx{iD=pec5CX3iI~igEev<_?Jg2p2*K&8 zsds0^qhDnvUN33hlegO9QEtjbb({v`16{tz`XO%Dqbi3F>*?we`6?zH<)I)#QdF7u zs>h6CgD`^}cl{V{Y*NBJBcyQkw?Xz5sbL7>o`p zDYdJQqMRbC9p-89HJEBOG=bE(0h`dEZT>zzGm}EcKtCmu&X6xDE^cXMg>nnrDjO3e zc+WBl&V=_Z7aeN$0)iqVB4TsiRy?%0xc}2II1q#*?9f++i(RI<5WiaZG6oM}XwUl=;ZDExAU8f~X!ErGPq1aQ zG!c2DSeKKD;;Hud0@?-$=g?cJKM<9XiA&*?Pg0PU9vd7)uCERMgUrlKCr8I_M2N4h zEjo+9yCJ}C(ss@vP^j9c$yJn#?z3&QBP+rjSZ>!Y+jgRx;cjWdWsgBS6+4!zM==Va&< zBN1oyms|_Bai5bW$W7fozOT}}4>khKA%F=R2Brb8U_DS)k|=pSTQ}F00xE6k9`!o% zhR|vpTvu$K=UKh!?Ccb03|8XE{XAB9>6?$nDM$opx^2mKmWH`RD zWw(8BIUp=LEbM(t%XOjF@%f+9y}R>0JM8QSC*x^}HpAzF8R-6dVNcaMlCb7^kITPj zKag+F3;jzb#&RKF&9c(Rcx)lVnmai1p;Jbf`AwVKJA{X#kh38&u7LY>`;ambHw`rv z6*Q|FrP2I#uiMPTLNw7yfb)d5`FvJr%0rz0xDPFpNv5#M_fdtf&Ej?4-mE?t^Md|Y ztVL)F;iNY6J5a&K%g=v5>2jRF=Cq2)=MEw7)uE5Vf>bk8!VBQm|N8>x4bL@RvS<+g z)xZ5R^wTV2!_qpS1|+4|NZ*-Uzu4P;;X!0w?6QZrL}>@5w*Do9QNcK?oBC9wFj`#GkU+w=#sx>4XG4Bsepy zgX_4{+}xCl_RE_uA<<$(gRXVw0_9c!WHg>sH zM~e^~1e?8|pI!Ttjag9pVL+Mli2c#Jbt@iNq%Rm5sU6ovRl=VGiHq&RfSspF|04C0 zq=`*T>_JFOOUofvy8ej?@4-6CZ2c2@dS4K+rVASljyBpP{v`H?a3yW;7!yKJKR1BffQ1m>ov4-y|PJtRQOPZO}V=IyKaLORH=T3yvObcNYQ z%i_iJSN~LgQI5`g{1wQja;5*XWE*_}lKZRC8v?6(AnMK!OQ-*j4yp%xw@MzKssd9(^=~kh?%PMn80Gk8| z2e3IIPSgH%&jeb_{WH>fV#VR^h(KOV z8BWx)>+`PK8#XX2t0^7Seb+7=Hql<`qmYeQ+umY(R_|yNe)dmSr)84mf=jOZA}q&c zEW%LWlB;VIL)vda8vAd#ztRgsjgqU7CnQr9-@AvhlA<9E{GK#2&Z}A zgIfIz8GTms-XSVBmJ_jRal|iO#zYH*3zHH6GJrguo)E_q z_OReOV7G+a;o}WL*5>qLa~zA5yJ92)!}MPssnz;Yym@s&?pdkAhl}RG!Vm0 zOzZ(C2}V4m&rjoeoy2~DhYv?wD_5ZVO2AxdU=XDl0ZoxR^i>U=pl4-Gk4!>UK+Qg{ zH~9Ipu!xBHV;Ny@$&WnEbzCnfC@7|oL9&bXw6f9&*`NxgzY@Z|jS$d*Me_Uis~*<{ zQHiV$Cnrg1>E(sW^;OpO#oF{F?^1N7DG#A2iYsgD4`X1E!UtDWS26xMgIVuOs|;~N z!aI5c{L)A|i^-DRzaJzVJvH@xT|Ur=djk6ZlBiXrjo12L@mYmNG0!}#&96QMZ-LSi zCxF%ToJmbunwrQ(ug)3Wernv|A0bnA?Ap@J%zqig{pN9|CVEatk#yrt&C^N^Q z&*wr$9}V2#>SyZ+{=YfBg4`??jRR;qYB*@z3Ggd=-lrfc{mPJ9;XY z{x;yMpH%bJ#+E2c0UCP5PAlIZ{rWZa)cR>#CBK{qYw6sX3j~`iISI*qAHrYkzo4=K zy$tjot(BQ9J7j-T_&mSDLX(QE<();Yq|QI4Y$g)I=e*)R?)Kxn+!d7_hHE0m8NiFB zr5PLk_`hrj{|SmYw>dfYuYdeEe-)_WJ`uW`O3l@^c*{w-4drxN$t6AxI+Js`tNrGL4PM+4?dk3wF!4Z;7txsXoLioZ2?N zSH_0u>(8fFXJgCc0-M6z)vHazSd0ON3SOswcwM~?L0Aluq_Y$I8%TZA($gg+B*5Ok z+ji;cvuA|-YrrOYdP^%SVjt!_$}!dTWy@2IY8d!kX5^3fi$a;Je=7Ypm}m)E+4w!f zgs`1Qc%z=YLoX05S@Ipi`5P80#)&2r&2@G1^;KIl(sKFaWV55g_)~w4QR3=PYY0ALXEA&+^q)cD$U{;s?bfhW&n=iDKm;j@Wg9&9} zK{#TF`Bo&46aZ2kZEb{!N?Kjh6Zi4*($>+T=Hs2QcbP z{^uXB&x42Td!W3@^o^Jg_rplKq{IgfKa*^}_Pd7u@Izodl3gS~A`5hJkq8tnC$$Jv zl*jGhB!A@MQs7}%6Zv&otpYlrJ^+!4V zJ(=fXKMiKHXM(FG`<>NDU2v)VzW~@jV9DnjjuW{?MvZmzvp)hgbaq(FVzor11Aswg zp!L0fMgQMOvm~oipMnxSKfaIK!Aqud@v}ZcdtV=cu*59$pF3no2oZW(;T&%{U}42= zAljyDmzA0wntjWA zY570}?f25lxjB+tBcJ{73ltMf$0_`BjShRu`^yRnqB+_TuiZP3L1}wDuv{@QvDjlV zBbk8`fxhO{6Q9;%5<}q__EiF@v_JJy@BW-AzaL1W8-ICAolRR$FN#8WSM11=|JEEP zkp^sN(cP6ZQN|RyVF~?cIM6U)t-`t+*x9+#1A)RPwTNdUfp@x=R6_fxkf^jZlSx+Z zAzs$LpH?R9}F z>!QDYWS13!y>HzzMWkm+ilwP3-B)Rya3fd95i`A%MCYhmdg35h+&DF{Pp}}$*xq6L z9A<43AcMq-V;;-ihU}wLNW154GuDB_uD15AJ+`$NYhXZ(LuJ}$&)iMC)Ngr-6@?R+ zK;53}W2z2AANLDdO+X-l-w4>`;2Z#Tz;O}ivRW7@#!4wXk&Yl{R4c)6j)Y((1+d2< zpeRndd9%csU#9rN_i^H@A4AYX4zy=b>xw96h}jj4v&(CfUF z^cNQ{T)?MMB*eg4VTdkEzQ!r4K{yapT2wTHNIDQr;y>*SzV9oujb|XVR=9fBhZyDi z+lYNw9HcTB-E9~{X<;PJyr^8ZKVV~XXmIV}5KN3bpT|n%;)HtP(B3qb+x&$ZvicVM z`)`j6{^z{ey>TTlQ{%lF)b4)adv4`&A*t-0Gc9d(cc^||T5>o^zs|#PCTIgagDSXl zYt3Nbsuc&{#SE$__5d%m3K&{*DUknMRQx|4#@}ORoX0mlIyl$Qt;1aKU!DFqQ}~3! zk+hfh79V?Kkwfa-5Dy*@%6XM)GyothBA^Z6oDdYfw{nN&xSwY5hmjN*B8` zj=qbISg~Sto6V2VdK5em-v0J}xc150HEa0`)jxjo-@MQVtQ!^;ZVKjTR$H8)(dDSV zQx%UbAOum0Aih?E3x|*w7ICQY+A@uM<%*{z3#BvDb{wV!Z$O1gCVE}u*z4Mv%cA1K ziUHabg1l?&atxNmrmH}=)gDXZiz~Hjh=9EWxmofCwDzki#q(QC<362aoNBS$yLFT<~>Y3da)aSeKh0 zVH^rxz9RN9JO>66nMTn-&)#%*KY^F@a2$2Wpn%jR67;w`;=k-N%r@s7F|Wm}DSLo|Bt~$U3Yz zA>idtfW0$T#|X%~B8FmH$ZQ(|^D^m3Av3ECn1Rcp5~32-v;csDBWD#jdJ_f3qo-8+ zp|7*UMpEA~RsSw~S4Rhhq>k92|d&Q?IB$P&r<71nn3p~N7>LmvB$|3IKKR#d9O zo{^SDVyNG5qY3#v+{b;oT_PCor?=xyxH9vN4skj%Phgs#EBzY6QFv)EqH1ev$y?rF z=AqN!c-jmc!;EWtk)L$6?vC6ar;dHz=WjPZ)__n$TVa?N_Dy|qpH5Cy4*UX#5^Rs4 zN4l2KR2Z`zET`#!^lsZNp=G^OYT;P94fRO8C`zzPcO_zv+u=!-s<&x5y+&WdOr)a! zUdL18NS8_hc-U^{;(~`wj>W9}`3hHrF=NjHYaLX@(!EoWPx+Hc2)3bQEK5jAcE(X210Ef%jipCTTuODFCM3TWh9LI{Yd^U<&|e z0%7uYadF_v9<<`Df`S$syXg@=9v<;ntsv|CM;oy3_FqvqO{1Czb*cq%5naOF6q58; z8%bqrs^CZtwIU{p0^t?narP)w*R6V z>keEj5AH^F+mm+w<*2u*WE^-axWO4xz=#2R$4i=e;|7P>3O1Cq1HpbE_kn^Zg~SOH zNmMTtMLJ9ofa$3~7S)j@K0K8sA(8cbcZ8%Yrc1~=6A@8+M5VvR=r#X4Zfuw&php5| zk5QhjxPx*Vv5Nx?mGxrR?{O|KP8OoBRrrj@`09=f8T22^xxA-QPaHX8dpB*qV-n=`=34gKr_Thehy~!08k3I7Xx^{>z(%z%b~VBV{X}BbiVMvQMh7ZF8l0|eY;4TT5__Z33jx5#u?h$Y zB-m`iAV|`E%_QSk=*2`PV#uptIE@Yj8vxXex}lC*jgcRLtDzs5R`6Q*GJ4|fS){C^ zFSJ ztbSHEg~uCW)^M&J0_(2+Lo#BuI(D(YTAh*%SJ~r~ysrI;Nl6f-sLl_rrmK%`kk&_f zC9I_Do0`Cf43kU16aseKjcy81G_mil2@xQQ&XIfwSci(XDlJ~g_rva<)hqHxBGn|} zJG5{K`&_PfxkiVDnN$Tm=pA=!7P#d432T#;f46$oDr|q-RL*^Hnfq(NElBa=H$zQ> zm}k_ZhVuSh*{uBb@spl4{7 zbE(0Qe{y1CmJ8)X!R;-EL$}Xe{M$`)q&4{=&CRbOWrTROPIy4wPTBB@S|*(;9D^P6 z{89k4C)hFT(ID&`qH%&ytIC+Yq0cH;tc$l7BYl{*mdO_$H=(`wpRHAQo=BG+)ji+r z?63?Do5OB-@susBG#@x~fiX);QDW8QYWC!s^W5Cr#^&a`mtNRo++}3cfhBcTpbo?# zOlbO47~&EVbkr#9$*kgrW}D?5&D9{)tQ37neh!TYfF5LmfFgyw>b%tR8{`Juq6kL= z=;JXq4yCdx27)q@D^?t|w$y^9)ApsJt}}M;WRqHe8BexwqP)j4HRb2$? zpPf_+lF3{M+%i>Yx>>Eqe4#SEX;-h;vmObL?54|C zOrP`^e(&yMA67^pu|_6O4>mm;tEn|MS<0FlTlyMFXQmo>PxPcUeU$y(GxsMMoKNqi zd)ZsOJhR8&Z@E3xPWi`}D*Nf@% zSIO<{Jof#as-ez0;m`=dJ5Q<#KlT2)JeO3Mzt@aqTW87a-*1uY%X*2Y=X!E$Gve0? z8(9Z`>QZdwRFhu*th(4cviuY3;yc)xrYf~Ge8XI3Q;4_hVxI8iTon%=x974)w>NWi z->#p!oP$Qzp{gR)9)zE-82syk9%R}Ih3$x5_aCAn*6;r75%72QQte?iwn^=C@f&W zdf;Y?1|=$?GA1Z>E$7{nejIY6nK{A>DdE=Y61^8V8}4Kki4yLVI{#Mj!cP3~-Svv? zTeTN%8R?v&^amLWh>@w?l=9%#CW+y-gejHO1nFwWVWB6nj97WOXXfNt$@pHs^sal` z*NiJ42(x;lm%*-`NG`okAzkWt8@(r;Q>Q2-FLiSkWiFlUdA*FR`T8sQv$D*yy2EV3 zo%LRXtXOaH=U!y*SUE3(ree&vWGsh{M3Wc6IlXhco0f#*^(#4Ub=0k74zlfnv<`=o9OoK+t?i7N2lWgnIks)P|)C645QNv#~*xM z|GQJfTiHgkF(cqv&l!J?>L>+ zIJ&Op6(c{gC>GX~zo@1ycJlX|>Ce`S2-^1#l~Q%fD;zv+1dcBfo1b$YRP5B!b>6OV zAher)e{oIsqxp%cE>Rhpydh(FbUDFnb%NhUcF`4L0^K?~+LT4^P=C(s0U7e{{O4j; z#FgUpTBmxG4+Bw~{UZGEFAcUk{oOjL;}wq|lkZSQh+Iu;5`y=&33W9S8xvlPqQ|vk zK@KG}J&Do4fzGe4(wlgriV$J|5W>6samgP1(rE6V1^uoMdm5F5Mn(L3H63%J4Ctfk zX6Hu9SH}ZW$1ivMNzrig;w_;`av%D$Qv)*x&xf(c@MG%zVnsyy-0>~eyL2gEjV<=; zPF@muy>GdNaw~g|?cQ6&7*S=TGuPC7Hs>g+dDZKT1zHhTO(`ziiSG%QxF(^>w}MaH2o&z?ShyvTWL%w^smt z#`dPH&bqAR^T+w3DkjWs@{UXBd0t*uN(|whcMuV-S643HbA(6bE!z!m*_jzlalmeb zoUXV8Z^iaY8W|hE(w7gMj4vSluzNqju=$O&A$5W^{wE}57XP5W+@(eOg{75Y%1zX%ZzbHEpxkidsd!yGS-A-0x!S=0| z`qc7lSJO6QZaFuRmljo)!nV}{$LqJ)eK|FnG#+Ssv~q6J@p<;{{OfFywAw#qQYv(1 jl!O{G=T>r4J~Qu^u+32 Date: Fri, 24 Aug 2018 15:18:16 -0400 Subject: [PATCH 0161/1853] PEP8 ofsite middleware --- scrapy/spidermiddlewares/offsite.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 310166cad..232e96cbb 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,7 +3,6 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ - import re import logging import warnings @@ -35,8 +34,9 @@ class OffsiteMiddleware(object): domain = urlparse_cached(x).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) - logger.debug("Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) self.stats.inc_value('offsite/domains', spider=spider) self.stats.inc_value('offsite/filtered', spider=spider) else: @@ -52,13 +52,15 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return re.compile('') # allow all by default + return re.compile('') # allow all by default url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): - warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) - - regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) + message = ("allowed_domains accepts only domains, not URLs. " + "Ignoring URL entry %s in allowed_domains." % domain) + warnings.warn(message, URLWarning) + domains = [re.escape(d) for d in allowed_domains if d is not None] + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider): From c02cfa574cc47d6b086cc66025ddef9f3174ac02 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 29 Aug 2018 11:21:55 +0000 Subject: [PATCH 0162/1853] remove comma --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index cf7b8db2f..ff07b9d55 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -285,7 +285,7 @@ request_reached_downloader .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request`, reached downloader. + Sent when a :class:`~scrapy.http.Request` reached downloader. The signal does not support returning deferreds from their handlers. From 8dbbbd13950dcb21dda759b073c64ffdca85c2d6 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Mon, 3 Sep 2018 20:07:37 +0300 Subject: [PATCH 0163/1853] Use request_cls attribute in contract definition --- docs/topics/contracts.rst | 5 +++-- scrapy/contracts/__init__.py | 15 +++++++++------ tests/test_contracts.py | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ada6fd227..70f20d4ed 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -86,8 +86,9 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used - if ``request_cls`` is not set on ``args``. + for request object. :class:`~scrapy.http.Request` is used by default, + but this can be changed with the ``request_cls`` attribute. + If multiple contracts in chain have this attribute defined, the last one is used. Must return the same or a modified version of it. diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 801c18e73..851a26a8e 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -4,7 +4,6 @@ from functools import wraps from inspect import getmembers from unittest import TestCase -from scrapy import FormRequest from scrapy.http import Request from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec @@ -50,14 +49,17 @@ class ContractsManager(object): def from_method(self, method, results): contracts = self.extract_contracts(method) if contracts: - # prepare request arguments - kwargs = {'callback': method} + request_cls = Request + for contract in contracts: + if contract.request_cls is not None: + request_cls = contract.request_cls + + # calculate request args + args, kwargs = get_spec(request_cls.__init__) + kwargs['callback'] = method for contract in contracts: kwargs = contract.adjust_request_args(kwargs) - request_cls = kwargs.pop('request_cls', Request) - - args, _ = get_spec(request_cls.__init__) args.remove('self') # check if all positional arguments are defined in kwargs @@ -98,6 +100,7 @@ class ContractsManager(object): class Contract(object): """ Abstract class for contracts """ + request_cls = None def __init__(self, method, *args): self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c35b068a4..fc5c94771 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -27,9 +27,9 @@ class ResponseMock(object): class CustomFormContract(Contract): name = 'custom_form' + request_cls = FormRequest def adjust_request_args(self, args): - args['request_cls'] = FormRequest args['formdata'] = {'name': 'scrapy'} return args From e65f7e0c91ccb16525bb318cb50339979387fdbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Sep 2018 10:49:46 -0300 Subject: [PATCH 0164/1853] Working POC for authenticating telnet console --- scrapy/extensions/telnet.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3024ddfaa..5e9fce7c5 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -22,6 +22,7 @@ from scrapy import signals from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp +from scrapy.utils.decorators import defers try: import guppy @@ -49,6 +50,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] + self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy') + self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy') self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -67,9 +70,25 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self): - telnet_vars = self._get_telnet_vars() - return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, - insults.ServerProtocol, manhole.Manhole, telnet_vars) + class Portal: + """An implementation of IPortal""" + @defers + def login(self_, credentials, mind, *interfaces): + if not (credentials.username == self.username + and credentials.checkPassword(self.password)): + raise ValueError("Invalid credentials") + + protocol = telnet.TelnetBootstrapProtocol( + insults.ServerProtocol, + manhole.Manhole, + self._get_telnet_vars() + ) + return (interfaces[0], protocol, lambda: None) + + return telnet.TelnetTransport( + telnet.AuthenticatingTelnetProtocol, + Portal() + ) def _get_telnet_vars(self): # Note: if you add entries here also update topics/telnetconsole.rst From eb64214c8a0053627e625debdb55373c8e17ef1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Sep 2018 10:54:40 -0300 Subject: [PATCH 0165/1853] Move telnetconsole settings defaults to scrapy defaults --- scrapy/extensions/telnet.py | 4 ++-- scrapy/settings/default_settings.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 5e9fce7c5..93342f225 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -50,8 +50,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy') - self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy') + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..2b7bc173c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { From 25ac4691b414e7c18a4e9dec3bb6a85563d2488d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 9 Aug 2018 03:32:46 +0500 Subject: [PATCH 0166/1853] require parsel 1.5+ --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 03b33d02d..0771aae3a 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -6,5 +6,5 @@ queuelib w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -parsel>=1.4 +parsel>=1.5 service_identity diff --git a/requirements-py3.txt b/requirements-py3.txt index b38c4cc09..5a5d4c95a 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -6,5 +6,5 @@ queuelib>=1.1.1 w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -parsel>=1.4 +parsel>=1.5 service_identity diff --git a/setup.py b/setup.py index c37919cda..8c47f67ce 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=1.4', + 'parsel>=1.5', 'PyDispatcher>=2.0.5', 'service_identity', ], From 0ccead6681c9e2bf1902cbc1d4bde543be7d73e7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Aug 2018 16:16:36 +0500 Subject: [PATCH 0167/1853] DOC more Python 3 in examples --- docs/topics/commands.rst | 6 ++--- docs/topics/items.rst | 6 ++--- docs/topics/jobs.rst | 4 ++-- docs/topics/loaders.rst | 14 ++++++------ docs/topics/selectors.rst | 48 +++++++++++++++++++-------------------- docs/topics/settings.rst | 2 +- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3088017cb..ef9c45196 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -458,9 +458,9 @@ Usage example:: >>> STATUS DEPTH LEVEL 1 <<< # Scraped Items ------------------------------------------------------------ - [{'name': u'Example item', - 'category': u'Furniture', - 'length': u'12 cm'}] + [{'name': 'Example item', + 'category': 'Furniture', + 'length': '12 cm'}] # Requests ----------------------------------------------------------------- [] diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 4423bbda2..ae44aecd3 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -86,7 +86,7 @@ Creating items :: >>> product = Product(name='Desktop PC', price=1000) - >>> print product + >>> print(product) Product(name='Desktop PC', price=1000) Getting field values @@ -161,11 +161,11 @@ Other common tasks Copying items:: >>> product2 = Product(product) - >>> print product2 + >>> print(product2) Product(name='Desktop PC', price=1000) >>> product3 = product2.copy() - >>> print product3 + >>> print(product3) Product(name='Desktop PC', price=1000) Creating dicts from items:: diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 06c7fff3d..8e1574376 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -84,7 +84,7 @@ So, for example, this won't work:: return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg)) def other_callback(self, response, somearg): - print "the argument passed is:", somearg + print("the argument passed is: %s" % somearg) But this will:: @@ -94,7 +94,7 @@ But this will:: def other_callback(self, response): somearg = response.meta['somearg'] - print "the argument passed is:", somearg + print("the argument passed is: %s" % somearg) If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index a895b535c..f3b6aa4a1 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -678,10 +678,10 @@ Here is a list of all built-in processors: >>> from scrapy.loader.processors import Join >>> proc = Join() >>> proc(['one', 'two', 'three']) - u'one two three' + 'one two three' >>> proc = Join('
') >>> proc(['one', 'two', 'three']) - u'one
two
three' + 'one
two
three' .. class:: Compose(\*functions, \**default_loader_context) @@ -744,9 +744,9 @@ Here is a list of all built-in processors: ... return None if x == 'world' else x ... >>> from scrapy.loader.processors import MapCompose - >>> proc = MapCompose(filter_world, unicode.upper) - >>> proc([u'hello', u'world', u'this', u'is', u'scrapy']) - [u'HELLO, u'THIS', u'IS', u'SCRAPY'] + >>> proc = MapCompose(filter_world, str.upper) + >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] As with the Compose processor, functions can receive Loader contexts, and constructor keyword arguments are used as default context values. See @@ -772,7 +772,7 @@ Here is a list of all built-in processors: >>> import json >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo")) >>> proc_single_json_str('{"foo": "bar"}') - u'bar' + 'bar' >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo'))) >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]') - [u'bar'] + ['bar'] diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8ac40c3cc..25c1f0aab 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -235,17 +235,17 @@ Here's an example used to extract image names from the :ref:`HTML code ` above:: >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') - [u'My image 1', - u'My image 2', - u'My image 3', - u'My image 4', - u'My image 5'] + ['My image 1', + 'My image 2', + 'My image 3', + 'My image 4', + 'My image 5'] There's an additional helper reciprocating ``.extract_first()`` for ``.re()``, named ``.re_first()``. Use it to extract just the first matching string:: >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') - u'My image 1' + 'My image 1' .. _topics-selectors-relative-xpaths: @@ -431,26 +431,26 @@ with groups of itemscopes and corresponding itemprops:: ... print " properties:", props.extract() ... print - current scope: [u'http://schema.org/Product'] - properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review'] + current scope: ['http://schema.org/Product'] + properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review'] - current scope: [u'http://schema.org/AggregateRating'] - properties: [u'ratingValue', u'reviewCount'] + current scope: ['http://schema.org/AggregateRating'] + properties: ['ratingValue', 'reviewCount'] - current scope: [u'http://schema.org/Offer'] - properties: [u'price', u'availability'] + current scope: ['http://schema.org/Offer'] + properties: ['price', 'availability'] - current scope: [u'http://schema.org/Review'] - properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + current scope: ['http://schema.org/Review'] + properties: ['name', 'author', 'datePublished', 'reviewRating', 'description'] - current scope: [u'http://schema.org/Rating'] - properties: [u'worstRating', u'ratingValue', u'bestRating'] + current scope: ['http://schema.org/Rating'] + properties: ['worstRating', 'ratingValue', 'bestRating'] - current scope: [u'http://schema.org/Review'] - properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + current scope: ['http://schema.org/Review'] + properties: ['name', 'author', 'datePublished', 'reviewRating', 'description'] - current scope: [u'http://schema.org/Rating'] - properties: [u'worstRating', u'ratingValue', u'bestRating'] + current scope: ['http://schema.org/Rating'] + properties: ['worstRating', 'ratingValue', 'bestRating'] >>> @@ -543,22 +543,22 @@ Example:: This gets all first ``
  • `` elements under whatever it is its parent:: >>> xp("//li[1]") - [u'
  • 1
    • ', u'
  • 4
    • '] + ['
  • 1
    • ', '
  • 4
    • '] And this gets the first ``
  • `` element in the whole document:: >>> xp("(//li)[1]") - [u'
  • 1
    • '] + ['
  • 1
    • '] This gets all first ``
  • `` elements under an ``
      `` parent:: >>> xp("//ul/li[1]") - [u'
    • 1
      • ', u'
    • 4
      • '] + ['
    • 1
      • ', '
    • 4
      • '] And this gets the first ``
    • `` element under an ``
        `` parent in the whole document:: >>> xp("(//ul/li)[1]") - [u'
      • 1
        • '] + ['
      • 1
        • '] When querying by class, consider using CSS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1f1217770..47b6cf13d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -871,7 +871,7 @@ LOG_STDOUT Default: ``False`` If ``True``, all standard output (and error) of your process will be redirected -to the log. For example if you ``print 'hello'`` it will appear in the Scrapy +to the log. For example if you ``print('hello')`` it will appear in the Scrapy log. .. setting:: LOG_SHORT_NAMES From 395d9d033a39ae9e82337ce50cffabd1ecedb702 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Aug 2018 16:16:57 +0500 Subject: [PATCH 0168/1853] add pytest temp files to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ff6e2ea65..7392ed31e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ htmlcov/ .pytest_cache/ .coverage.* .cache/ +.pytest_cache/ # Windows Thumbs.db From ca27010d4f8c35d1c98259e56be797a18b044304 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Aug 2018 16:22:56 +0500 Subject: [PATCH 0169/1853] DOC .extract_first() -> .get() --- docs/intro/overview.rst | 6 ++--- docs/intro/tutorial.rst | 49 +++++++++++++++++++++-------------------- docs/topics/shell.rst | 2 +- docs/topics/spiders.rst | 10 ++++----- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6f1c2c43f..9d7c94d39 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.xpath('span/small/text()').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.xpath('span/small/text()').get(), } - next_page = response.css('li.next a::attr("href")').extract_first() + next_page = response.css('li.next a::attr("href")').get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fa6dc274d..a24cf0f5b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -269,23 +269,24 @@ The other thing is that the result of calling ``.extract()`` is a list, because we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When you know you just want the first result, as in this case, you can do:: - >>> response.css('title::text').extract_first() + >>> response.css('title::text').get() 'Quotes to Scrape' As an alternative, you could've written:: - >>> response.css('title::text')[0].extract() + >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.extract_first()`` avoids an ``IndexError`` and returns -``None`` when it doesn't find any element matching the selection. +However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` +instance avoids an ``IndexError`` and returns ``None`` when it doesn't +find any element matching the selection. There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.Selector.extract` and -:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use +:meth:`~scrapy.selector.SelectorList.get` methods, you can also use the :meth:`~scrapy.selector.Selector.re` method to extract using `regular expressions`:: @@ -314,7 +315,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: >>> response.xpath('//title') [] - >>> response.xpath('//title/text()').extract_first() + >>> response.xpath('//title/text()').get() 'Quotes to Scrape' XPath expressions are very powerful, and are the foundation of Scrapy @@ -383,10 +384,10 @@ variable, so that we can run our CSS selectors directly on a particular quote:: Now, let's extract ``title``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").extract_first() + >>> title = quote.css("span.text::text").get() >>> title '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' - >>> author = quote.css("small.author::text").extract_first() + >>> author = quote.css("small.author::text").get() >>> author 'Albert Einstein' @@ -401,8 +402,8 @@ Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): - ... text = quote.css("span.text::text").extract_first() - ... author = quote.css("small.author::text").extract_first() + ... text = quote.css("span.text::text").get() + ... author = quote.css("small.author::text").get() ... tags = quote.css("div.tags a.tag::text").extract() ... print(dict(text=text, author=author, tags=tags)) {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} @@ -434,8 +435,8 @@ in the callback, as you can see below:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -508,14 +509,14 @@ markup: We can try extracting it in the shell:: - >>> response.css('li.next a').extract_first() + >>> response.css('li.next a').get() 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, Scrapy supports a CSS extension that let's you select the attribute contents, like this:: - >>> response.css('li.next a::attr(href)').extract_first() + >>> response.css('li.next a::attr(href)').get() '/page/2/' Let's see now our spider modified to recursively follow the link to the next @@ -533,12 +534,12 @@ page, extracting data from it:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), 'tags': quote.css('div.tags a.tag::text').extract(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -584,12 +585,12 @@ As a shortcut for creating Request objects you can use def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('span small::text').get(), 'tags': quote.css('div.tags a.tag::text').extract(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -641,7 +642,7 @@ this time for scraping author information:: def parse_author(self, response): def extract_with_css(query): - return response.css(query).extract_first().strip() + return response.css(query).get().strip() yield { 'name': extract_with_css('h3.author-title::text'), @@ -710,11 +711,11 @@ with a specific tag, building the URL based on the argument:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 11ab199f2..9de6abef7 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -179,7 +179,7 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: - >>> response.xpath('//title/text()').extract_first() + >>> response.xpath('//title/text()').get() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("https://reddit.com") diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 697732b47..4505b7315 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules:: self.logger.info('Hi, this is an item page! %s', response.url) item = scrapy.Item() item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = response.xpath('//td[@id="item_name"]/text()').extract() - item['description'] = response.xpath('//td[@id="item_description"]/text()').extract() + item['name'] = response.xpath('//td[@id="item_name"]/text()').get() + item['description'] = response.xpath('//td[@id="item_description"]/text()').get() return item @@ -548,9 +548,9 @@ These spiders are pretty easy to use, let's have a look at one example:: self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract())) item = TestItem() - item['id'] = node.xpath('@id').extract() - item['name'] = node.xpath('name').extract() - item['description'] = node.xpath('description').extract() + item['id'] = node.xpath('@id').get() + item['name'] = node.xpath('name').get() + item['description'] = node.xpath('description').get() return item Basically what we did up there was to create a spider that downloads a feed from From d32c4deaa99bf0155542d1b508b5535065b5d7a9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Aug 2018 16:23:31 +0500 Subject: [PATCH 0170/1853] DOC update Scrapy selectors tutorial to match parsel's tutorial better --- docs/topics/selectors.rst | 471 +++++++++++++++++++++++++++----------- 1 file changed, 340 insertions(+), 131 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 25c1f0aab..00158ecf1 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -6,7 +6,7 @@ Selectors When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to -achieve this: +achieve this, such as: * `BeautifulSoup`_ is a very popular web scraping library among Python programmers which constructs a Python object based on the structure of the @@ -25,8 +25,9 @@ either by `XPath`_ or `CSS`_ expressions. used with HTML. `CSS`_ is a language for applying styles to HTML documents. It defines selectors to associate those styles with specific HTML elements. -Scrapy selectors are built over the `lxml`_ library, which means they're very -similar in speed and parsing accuracy. +Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library +under the hood. It means Scrapy selectors are very similar in speed and +parsing accuracy to lxml. This page explains how selectors work and describes their API which is very small and simple, unlike the `lxml`_ API which is much bigger because the @@ -42,7 +43,7 @@ For a complete reference of the selectors API see .. _cssselect: https://pypi.python.org/pypi/cssselect/ .. _XPath: https://www.w3.org/TR/xpath .. _CSS: https://www.w3.org/TR/selectors - +.. _parsel: https://parsel.readthedocs.io/ Using selectors =============== @@ -63,21 +64,32 @@ input type:: Constructing from text:: >>> body = 'good' - >>> Selector(text=body).xpath('//span/text()').extract() - [u'good'] + >>> Selector(text=body).xpath('//span/text()').get() + 'good' Constructing from response:: >>> response = HtmlResponse(url='http://example.com', body=body) - >>> Selector(response=response).xpath('//span/text()').extract() - [u'good'] + >>> Selector(response=response).xpath('//span/text()').get() + 'good' For convenience, response objects expose a selector on `.selector` attribute, -it's totally OK to use this shortcut when possible:: +it's totally OK to use this shortcut when possible. By using it you can +ensure the response body is parsed only once:: - >>> response.selector.xpath('//span/text()').extract() - [u'good'] + >>> response.selector.xpath('//span/text()').get() + 'good' +Querying responses using XPath and CSS is so common that responses include two +more shortcuts: ``response.xpath()`` and ``response.css()``:: + + >>> response.xpath('//span/text()').get() + 'good' + >>> response.css('span::text').get() + 'good' + +Usually there is no need to construct Scrapy selectors manually because of +these shortcuts. Using selectors --------------- @@ -90,7 +102,7 @@ documentation server: .. _topics-selectors-htmlcode: -Here's its HTML code: +For the sake of completeness, here's its full HTML code: .. literalinclude:: ../_static/selectors-sample1.html :language: html @@ -111,90 +123,179 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse So, by looking at the :ref:`HTML code ` of that page, let's construct an XPath for selecting the text inside the title tag:: - >>> response.selector.xpath('//title/text()') - [] - -Querying responses using XPath and CSS is so common that responses include two -convenience shortcuts: ``response.xpath()`` and ``response.css()``:: - >>> response.xpath('//title/text()') - [] - >>> response.css('title::text') - [] + [] + +To actually extract the textual data, you must call the selector ``.get()`` +or ``.getall()`` methods, as follows:: + + >>> response.xpath('//title/text()').getall() + ['Example website'] + >>> response.xpath('//title/text()').get() + 'Example website' + +``.get()`` always returns a single result; if there are several matches, +content of a first match is returned; if there are no matches, None +is returned. ``.getall()`` returns a list with all results. + +Notice that CSS selectors can select text or attribute nodes using CSS3 +pseudo-elements:: + + >>> selector.css('title::text').get() + 'Example website' As you can see, ``.xpath()`` and ``.css()`` methods return a :class:`~scrapy.selector.SelectorList` instance, which is a list of new selectors. This API can be used for quickly selecting nested data:: - >>> response.css('img').xpath('@src').extract() - [u'image1_thumb.jpg', - u'image2_thumb.jpg', - u'image3_thumb.jpg', - u'image4_thumb.jpg', - u'image5_thumb.jpg'] + >>> response.css('img').xpath('@src').getall() + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] -To actually extract the textual data, you must call the selector ``.extract()`` -method, as follows:: +If you want to extract only the first matched element, you can call the +selector ``.get()`` (or its alias ``.extract_first()`` commonly used in +previous Scrapy versions):: - >>> response.xpath('//title/text()').extract() - [u'Example website'] + >>> response.xpath('//div[@id="images"]/a/text()').get() + 'Name: My image 1 ' -If you want to extract only first matched element, you can call the selector ``.extract_first()`` +It returns ``None`` if no element was found:: - >>> response.xpath('//div[@id="images"]/a/text()').extract_first() - u'Name: My image 1 ' - -It returns ``None`` if no element was found: - - >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None + >>> response.xpath('//div[@id="not-exists"]/text()').get() is None True -A default return value can be provided as an argument, to be used instead of ``None``: +A default return value can be provided as an argument, to be used instead +of ``None``: - >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found') + >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found') 'not-found' -Notice that CSS selectors can select text or attribute nodes using CSS3 -pseudo-elements:: +Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes +using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:: - >>> response.css('title::text').extract() - [u'Example website'] + >>> [img.attrib['src'] for img in response.css('img')] + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] + +As a shortcut, ``.attrib`` is also available on SelectorList directly; +it returns attributes for the first matching element:: + + >>> response.css('img').attrib['src'] + 'image1_thumb.jpg' + +This is most useful when only a single result is expected, e.g. when selecting +by id, or selecting unique elements on a web page:: + + >>> response.css('base').attrib['href'] + 'http://example.com/' Now we're going to get the base URL and some image links:: - >>> response.xpath('//base/@href').extract() - [u'http://example.com/'] + >>> response.xpath('//base/@href').get() + 'http://example.com/' - >>> response.css('base::attr(href)').extract() - [u'http://example.com/'] + >>> response.css('base::attr(href)').get() + 'http://example.com/' - >>> response.xpath('//a[contains(@href, "image")]/@href').extract() - [u'image1.html', - u'image2.html', - u'image3.html', - u'image4.html', - u'image5.html'] + >>> response.css('base').attrib['href'] + 'http://example.com/' - >>> response.css('a[href*=image]::attr(href)').extract() - [u'image1.html', - u'image2.html', - u'image3.html', - u'image4.html', - u'image5.html'] + >>> response.xpath('//a[contains(@href, "image")]/@href').getall() + ['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] - >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract() - [u'image1_thumb.jpg', - u'image2_thumb.jpg', - u'image3_thumb.jpg', - u'image4_thumb.jpg', - u'image5_thumb.jpg'] + >>> response.css('a[href*=image]::attr(href)').getall() + ['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] - >>> response.css('a[href*=image] img::attr(src)').extract() - [u'image1_thumb.jpg', - u'image2_thumb.jpg', - u'image3_thumb.jpg', - u'image4_thumb.jpg', - u'image5_thumb.jpg'] + >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall() + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] + + >>> response.css('a[href*=image] img::attr(src)').getall() + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] + +.. _topics-selectors-css-extensions: + +Extensions to CSS Selectors +--------------------------- + +Per W3C standards, `CSS selectors`_ do not support selecting text nodes +or attribute values. +But selecting these is so essential in a web scraping context +that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**: + +* to select text nodes, use ``::text`` +* to select attribute values, use ``::attr(name)`` where *name* is the + name of the attribute that you want the value of + +.. warning:: + These pseudo-elements are Scrapy-/Parsel-specific. + They will most probably not work with other libraries like + `lxml`_ or `PyQuery`_. + +.. _PyQuery: https://pypi.python.org/pypi/pyquery + +Examples: + +* ``title::text`` selects children text nodes of a descendant ```` element:: + + >>> response.css('title::text').get() + 'Example website' + +* ``*::text`` selects all descendant text nodes of the current selector context:: + + >>> response.css('#images *::text').getall() + ['\n ', + 'Name: My image 1 ', + '\n ', + 'Name: My image 2 ', + '\n ', + 'Name: My image 3 ', + '\n ', + 'Name: My image 4 ', + '\n ', + 'Name: My image 5 ', + '\n '] + +* ``a::attr(href)`` selects the *href* attribute value of descendant links:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] + +.. note:: + You cannot chain these pseudo-elements. But in practice it would not + make much sense: text nodes do not have attributes, and attribute values + are string values already and do not have children nodes. + +.. note:: + See also: :ref:`selecting-attributes`. + + +.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors .. _topics-selectors-nesting-selectors: @@ -206,22 +307,65 @@ of the same type, so you can call the selection methods for those selectors too. Here's an example:: >>> links = response.xpath('//a[contains(@href, "image")]') - >>> links.extract() - [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>', - u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>', - u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>', - u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>', - u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] + >>> links.getall() + ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>', + '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>', + '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>', + '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>', + '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] >>> for index, link in enumerate(links): - ... args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract()) - ... print 'Link number %d points to url %s and image %s' % args + ... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) + ... print('Link number %d points to url %r and image %r' % args) - Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] - Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg'] - Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg'] - Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] - Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] + Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' + Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' + Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' + Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg' + Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg' + +.. _selecting-attributes: + +Selecting element attributes +---------------------------- + +There are several ways to get a value of an attribute. First, one can use +XPath syntax:: + + >>> response.xpath("//a/@href").getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +XPath syntax has a few advantages: it is a standard XPath feature, and +``@attributes`` can be used in other parts of an XPath expression - e.g. +it is possible to filter by attribute value. + +Scrapy also provides an extension to CSS selectors (``::attr(...)``) +which allows to get attribute values:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +In addition to that, there is a ``.attrib`` property of Selector. +You can use it if you prefer to lookup attributes in Python +code, without using XPaths or CSS extensions:: + + >>> [a.attrib['href'] for a in response.css('a')] + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +This property is also available on SelectorList; it returns a dictionary +with attributes of a first matching element. It is convenient to use when +a selector is expected to give a single result (e.g. when selecting by element +ID, or when selecting an unique element on a page):: + + >>> response.css('base').attrib + {'href': 'http://example.com/'} + >>> response.css('base').attrib['href'] + 'http://example.com/' + +``.attrib`` property of an empty SelectorList is empty:: + + >>> response.css('foo').attrib + {} Using selectors with regular expressions ---------------------------------------- @@ -241,8 +385,9 @@ Here's an example used to extract image names from the :ref:`HTML code 'My image 4', 'My image 5'] -There's an additional helper reciprocating ``.extract_first()`` for ``.re()``, -named ``.re_first()``. Use it to extract just the first matching string:: +There's an additional helper reciprocating ``.get()`` (and its +alias ``.extract_first()``) for ``.re()``, named ``.re_first()``. +Use it to extract just the first matching string:: >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') 'My image 1' @@ -266,17 +411,17 @@ it actually extracts all ``<p>`` elements from the document, not only those inside ``<div>`` elements:: >>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document - ... print p.extract() + ... print(p.get()) This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):: >>> for p in divs.xpath('.//p'): # extracts all <p> inside - ... print p.extract() + ... print(p.get()) Another common case would be to extract all direct ``<p>`` children:: >>> for p in divs.xpath('p'): - ... print p.extract() + ... print(p.get()) For more details about relative XPaths see the `Location Paths`_ section in the XPath specification. @@ -298,14 +443,14 @@ Here's an example to match an element based on its "id" attribute value, without hard-coding it (that was shown previously):: >>> # `$val` used in the expression, a `val` argument needs to be passed - >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first() - u'Name: My image 1 ' + >>> response.xpath('//div[@id=$val]/a/text()', val='images').get() + 'Name: My image 1 ' Here's another example, to find the "id" attribute of a ``<div>`` tag containing five ``<a>`` children (here we pass the value ``5`` as an integer):: - >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first() - u'images' + >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get() + 'images' All variable references must have a binding value when calling ``.xpath()`` (otherwise you'll get a ``ValueError: XPath error:`` exception). @@ -314,13 +459,12 @@ This is done by passing as many named arguments as necessary. `parsel`_, the library powering Scrapy selectors, has more details and examples on `XPath variables`_. -.. _parsel: https://parsel.readthedocs.io/ .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions Using EXSLT extensions ---------------------- -Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions +Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions and come with these pre-registered namespaces to use in XPath expressions: @@ -340,7 +484,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit:: >>> from scrapy import Selector - >>> doc = """ + >>> doc = u""" ... <div> ... <ul> ... <li class="item-0"><a href="link1.html">first item</a></li> @@ -352,10 +496,10 @@ Example selecting links in list item with a "class" attribute ending with a digi ... </div> ... """ >>> sel = Selector(text=doc, type="html") - >>> sel.xpath('//li//@href').extract() - [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract() - [u'link1.html', u'link2.html', u'link4.html', u'link5.html'] + >>> sel.xpath('//li//@href').getall() + ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] + >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() + ['link1.html', 'link2.html', 'link4.html', 'link5.html'] >>> .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular @@ -372,7 +516,7 @@ extracting text elements for example. Example extracting microdata (sample content taken from http://schema.org/Product) with groups of itemscopes and corresponding itemprops:: - >>> doc = """ + >>> doc = u""" ... <div itemscope itemtype="http://schema.org/Product"> ... <span itemprop="name">Kenmore White 17" Microwave</span> ... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' /> @@ -424,12 +568,12 @@ with groups of itemscopes and corresponding itemprops:: ... """ >>> sel = Selector(text=doc, type="html") >>> for scope in sel.xpath('//div[@itemscope]'): - ... print "current scope:", scope.xpath('@itemtype').extract() + ... print("current scope:", scope.xpath('@itemtype').getall()) ... props = scope.xpath(''' ... set:difference(./descendant::*/@itemprop, ... .//*[@itemscope]/*/@itemprop)''') - ... print " properties:", props.extract() - ... print + ... print(" properties: %s" % (props.getall())) + ... print("") current scope: ['http://schema.org/Product'] properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review'] @@ -493,27 +637,27 @@ Example:: Converting a *node-set* to string:: - >>> sel.xpath('//a//text()').extract() # take a peek at the node-set - [u'Click here to go to the ', u'Next Page'] - >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string - [u'Click here to go to the '] + >>> sel.xpath('//a//text()').getall() # take a peek at the node-set + ['Click here to go to the ', 'Next Page'] + >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string + ['Click here to go to the '] A *node* converted to a string, however, puts together the text of itself plus of all its descendants:: - >>> sel.xpath("//a[1]").extract() # select the first node - [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>'] - >>> sel.xpath("string(//a[1])").extract() # convert it to string - [u'Click here to go to the Next Page'] + >>> sel.xpath("//a[1]").getall() # select the first node + ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] + >>> sel.xpath("string(//a[1])").getall() # convert it to string + ['Click here to go to the Next Page'] So, using the ``.//text()`` node-set won't select anything in this case:: - >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract() + >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall() [] But using the ``.`` to mean the node, works:: - >>> sel.xpath("//a[contains(., 'Next Page')]").extract() - [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>'] + >>> sel.xpath("//a[contains(., 'Next Page')]").getall() + ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] .. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions @@ -538,7 +682,7 @@ Example:: ....: <li>5</li> ....: <li>6</li> ....: </ul>""") - >>> xp = lambda x: sel.xpath(x).extract() + >>> xp = lambda x: sel.xpath(x).getall() This gets all first ``<li>`` elements under whatever it is its parent:: @@ -578,12 +722,59 @@ you can just select by class using CSS and then switch to XPath when needed:: >>> from scrapy import Selector >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>') - >>> sel.css('.shout').xpath('./time/@datetime').extract() - [u'2014-07-23 19:00'] + >>> sel.css('.shout').xpath('./time/@datetime').getall() + ['2014-07-23 19:00'] This is cleaner than using the verbose XPath trick shown above. Just remember to use the ``.`` in the XPath expressions that will follow. +.. _old-extraction-api: + +extract() and extract_first() +----------------------------- + +If you're a long-time Scrapy user, you're probably familiar +with ``.extract()`` and ``.extract_first()`` selector methods. These methods +are still supported by Scrapy, there are no plans to deprecate them. + +However, Scrapy usage docs are now written using ``.get()`` and +``.getall()`` methods. We feel that these new methods result in a more concise +and readable code. + +The following examples show how these methods map to each other. + +1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:: + + >>> response.css('a::attr(href)').get() + 'image1.html' + >>> response.css('a::attr(href)').extract_first() + 'image1.html' + +2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + >>> response.css('a::attr(href)').extract() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +2. ``Selector.get()`` is the same as ``Selector.extract()``:: + + >>> response.css('a::attr(href)')[0].get() + 'image1.html' + >>> response.css('a::attr(href)')[0].extract() + 'image1.html' + +4. For consistency, there is also ``Selector.getall()``, which returns a list:: + + >>> response.css('a::attr(href)')[0].getall() + ['image1.html'] + +So, the main difference is that output of ``.get()`` and ``.getall()`` methods +is more predictable: ``.get()`` always returns a single result, ``.getall()`` +always returns a list of all extracted results. With ``.extract()`` method +it was not always obvious if a result is a list or not; to get a single +result either ``.extract()`` or ``.extract_first()`` should be called. + .. _topics-selectors-ref: @@ -718,10 +909,12 @@ SelectorList objects their results flattened, as a list of unicode strings. +.. _selector-examples-html: + Selector examples on HTML response ---------------------------------- -Here's a couple of :class:`Selector` examples to illustrate several concepts. +Here are some :class:`Selector` examples to illustrate several concepts. In all cases, we assume there is already a :class:`Selector` instantiated with a :class:`~scrapy.http.HtmlResponse` object like this:: @@ -735,20 +928,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: 2. Extract the text of all ``<h1>`` elements from an HTML response body, returning a list of unicode strings:: - sel.xpath("//h1").extract() # this includes the h1 tag - sel.xpath("//h1/text()").extract() # this excludes the h1 tag + sel.xpath("//h1").getall() # this includes the h1 tag + sel.xpath("//h1/text()").getall() # this excludes the h1 tag 3. Iterate over all ``<p>`` tags and print their class attribute:: for node in sel.xpath("//p"): - print node.xpath("@class").extract() + print(node.attrib['class']) + + +.. _selector-examples-xml: Selector examples on XML response --------------------------------- -Here's a couple of examples to illustrate several concepts. In both cases we -assume there is already a :class:`Selector` instantiated with an -:class:`~scrapy.http.XmlResponse` object like this:: +Here are some examples to illustrate concepts for :class:`Selector` objects +instantiated with an :class:`~scrapy.http.XmlResponse` object:: sel = Selector(xml_response) @@ -761,7 +956,7 @@ assume there is already a :class:`Selector` instantiated with an a namespace:: sel.register_namespace("g", "http://base.google.com/ns/1.0") - sel.xpath("//g:price").extract() + sel.xpath("//g:price").getall() .. _removing-namespaces: @@ -781,6 +976,20 @@ First, we open the shell with the url we want to scrape:: $ scrapy shell https://github.com/blog.atom +.. highlight:: xml + +This is how the file starts:: + + <?xml version="1.0" encoding="UTF-8"?> + <feed xml:lang="en-US" + xmlns="http://www.w3.org/2005/Atom" + xmlns:media="http://search.yahoo.com/mrss/"> + <id>tag:github.com,2008:/blog</id> + ... + +You can see two namespace declarations: a default "http://www.w3.org/2005/Atom" +and another one using the "media:" prefix for "http://search.yahoo.com/mrss/". + .. highlight:: python Once in the shell we can try selecting all ``<link>`` objects and see that it @@ -794,8 +1003,8 @@ nodes can be accessed directly by their names:: >>> response.selector.remove_namespaces() >>> response.xpath("//link") - [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>, - <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>, + [<Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>, + <Selector xpath='//link' data='<link xmlns="http://www.w3.org/2005/Atom'>, ... If you wonder why the namespace removal procedure isn't always called by default @@ -803,8 +1012,8 @@ instead of having to call it manually, this is because of two reasons, which, in of relevance, are: 1. Removing namespaces requires to iterate and modify all nodes in the - document, which is a reasonably expensive operation to perform for all - documents crawled by Scrapy + document, which is a reasonably expensive operation to perform by default + for all documents crawled by Scrapy 2. There could be some cases where using namespaces is actually required, in case some element names clash between namespaces. These cases are very rare From 09fd6c2a816ebec175ff3ffd945ce3263c9f1570 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 03:59:58 +0500 Subject: [PATCH 0171/1853] DOC unlink Firefox & Firebug sections from the tutorial for now. See https://github.com/scrapy/scrapy/issues/3373 and https://github.com/scrapy/scrapy/issues/3372 for motivation. --- docs/intro/tutorial.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a24cf0f5b..453f5114f 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -299,7 +299,8 @@ expressions`:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools (see section about :ref:`topics-developer-tools`). +You can use your browser developer tools to inspect the HTML and come up +with a selector (see section about :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. From 2c48d156db7d7846d80fa379dfb33552ac5b5a85 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:01:17 +0500 Subject: [PATCH 0172/1853] DOC cleanup references in tutorials: * remove unused link * fix ReST syntax * fix a link to regular expression docs --- docs/intro/tutorial.rst | 3 +-- docs/topics/selectors.rst | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 453f5114f..92d1065af 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -288,7 +288,7 @@ to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.Selector.extract` and :meth:`~scrapy.selector.SelectorList.get` methods, you can also use the :meth:`~scrapy.selector.Selector.re` method to extract using `regular -expressions`:: +expressions`_:: >>> response.css('title::text').re(r'Quotes.*') ['Quotes to Scrape'] @@ -740,4 +740,3 @@ modeling the scraped data. If you prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON -.. _dirbot: https://github.com/scrapy/dirbot diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 00158ecf1..cf1b0ba22 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -251,7 +251,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**: .. warning:: These pseudo-elements are Scrapy-/Parsel-specific. They will most probably not work with other libraries like - `lxml`_ or `PyQuery`_. + `lxml`_ or `PyQuery`_. .. _PyQuery: https://pypi.python.org/pypi/pyquery From 53da56c8dcd6edba12284ed5d125ddd1dcf6f99c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:17:55 +0500 Subject: [PATCH 0173/1853] TST update tests to use get/getall/attrib instead of extract --- tests/test_command_shell.py | 2 +- tests/test_http_response.py | 34 +++++++++++----------- tests/test_loader.py | 4 +-- tests/test_pipeline_crawl.py | 2 +- tests/test_selector.py | 24 +++++++-------- tests/test_spider.py | 8 ++--- tests/test_utils_iterators.py | 55 ++++++++++++++++++----------------- 7 files changed, 66 insertions(+), 63 deletions(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 3e27d6abd..36baacfbd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'response.xpath("//p[@class=\'one\']/text()").get()' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), b'Works') diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 820758dc9..3b90e3dac 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//title/text()").extract(), + response.selector.xpath("//title/text()").getall(), [u'Some page'] ) self.assertEqual( - response.selector.css("title::text").extract(), + response.selector.css("title::text").getall(), [u'Some page'] ) self.assertEqual( @@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//title/text()").extract(), - response.selector.xpath("//title/text()").extract(), + response.xpath("//title/text()").getall(), + response.selector.xpath("//title/text()").getall(), ) self.assertEqual( - response.css("title::text").extract(), - response.selector.css("title::text").extract(), + response.css("title::text").getall(), + response.selector.css("title::text").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), - response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(), + response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).extract(), - response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + pclass="content", pcount=1).getall(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) def test_urljoin_with_base_url(self): @@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//elem/text()").extract(), + response.selector.xpath("//elem/text()").getall(), [u'value'] ) @@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//elem/text()").extract(), - response.selector.xpath("//elem/text()").extract(), + response.xpath("//elem/text()").getall(), + response.selector.xpath("//elem/text()").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), ) response.selector.register_namespace('s2', 'http://scrapy.org') self.assertEqual( - response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s2:elem/text()").extract(), + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s2:elem/text()").getall(), ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 3b5714058..8b58e4dbd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>']) @@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_css("header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>']) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5985a6f3e..fb72c9d6d 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider): for href in response.xpath(''' //table[thead/tr/th="Filename"] /tbody//a/@href - ''').extract()], + ''').getall()], } yield item diff --git a/tests/test_selector.py b/tests/test_selector.py index 526660cc8..bc4baf7ea 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase): for x in xl: assert isinstance(x, Selector) - self.assertEqual(sel.xpath('//input').extract(), - [x.extract() for x in sel.xpath('//input')]) + self.assertEqual(sel.xpath('//input').getall(), + [x.get() for x in sel.xpath('//input')]) - self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")], + self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(), + self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), [u'xpathrules']) - self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) def test_root_base_url(self): @@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase): text = b'<div><img src="a.jpg"><p>Hello</div>' sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') - self.assertEqual(sel.xpath("//div").extract(), + self.assertEqual(sel.xpath("//div").getall(), [u'<div><img src="a.jpg"><p>Hello</p></img></div>']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') - self.assertEqual(sel.xpath("//div").extract(), + self.assertEqual(sel.xpath("//div").getall(), [u'<div><img src="a.jpg"><p>Hello</p></div>']) def test_http_header_encoding_precedence(self): @@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(), + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence - r1 = TextResponse('http://www.example.com', \ - body=b'<html><p>an Jos\xe9 de</p><html>', \ + r1 = TextResponse('http://www.example.com', + body=b'<html><p>an Jos\xe9 de</p><html>', encoding='utf-8') - Selector(r1).xpath('//text()').extract() + Selector(r1).xpath('//text()').getall() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" diff --git a/tests/test_spider.py b/tests/test_spider.py index 929e0fea8..f26da2334 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest): def parse_node(self, response, selector): yield { - 'loc': selector.xpath('a:loc/text()').extract(), - 'updated': selector.xpath('b:updated/text()').extract(), - 'other': selector.xpath('other/@value').extract(), - 'custom': selector.xpath('other/@b:custom').extract(), + 'loc': selector.xpath('a:loc/text()').getall(), + 'updated': selector.xpath('b:updated/text()').getall(), + 'other': selector.xpath('other/@value').getall(), + 'custom': selector.xpath('other/@b:custom').getall(), } for iterator in ('iternodes', 'xml'): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f953076b8..00eb78068 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in self.xmliter(response, 'product'): - attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract())) + attrs.append(( + x.attrib['id'], + x.xpath("name/text()").extract(), + x.xpath("./type/text()").extract())) self.assertEqual(attrs, - [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])]) def test_xmliter_unusual_node(self): body = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase): </root> """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').extract() + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) @@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase): attrs = [] for x in self.xmliter(r, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + attrs.append((x.attrib['id'], + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath(u'./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + [(u'26', [u'-'], [u'80']), + (u'21', [u'Ab'], [u'76']), + (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" - self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')], + self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) def test_xmliter_namespaces(self): @@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase): node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) - self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1']) - self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) - self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1']) - self.assertEqual(node.xpath('g:price/text()').extract(), ['400']) - self.assertEqual(node.xpath('image_link/text()').extract(), []) - self.assertEqual(node.xpath('id/text()').extract(), []) - self.assertEqual(node.xpath('price/text()').extract(), []) + self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) + self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) + self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) + self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) + self.assertEqual(node.xpath('image_link/text()').getall(), []) + self.assertEqual(node.xpath('id/text()').getall(), []) + self.assertEqual(node.xpath('price/text()').getall(), []) def test_xmliter_exception(self): body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" @@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase): body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( - next(self.xmliter(response, 'item')).extract(), + next(self.xmliter(response, 'item')).get(), u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>' ) @@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase): namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') node = next(namespace_iter) - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) node = next(namespace_iter) - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) + self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg']) def test_xmliter_namespaces_prefix(self): body = b"""\ @@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase): my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h') node = next(my_iter) - self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2) - self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples']) - self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas']) + self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2) + self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples']) + self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas']) my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f') node = next(my_iter) - self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table']) + self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table']) def test_xmliter_objtype_exception(self): i = self.xmliter(42, 'product') From 8c29be606c1be71059b454e41b87354e22569423 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:18:29 +0500 Subject: [PATCH 0174/1853] update spider templates to use .get --- scrapy/templates/spiders/crawl.tmpl | 10 +++++----- scrapy/templates/spiders/xmlfeed.tmpl | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 802cb88a1..878425125 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -14,8 +14,8 @@ class $classname(CrawlSpider): ) def parse_item(self, response): - i = {} - #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() - #i['name'] = response.xpath('//div[@id="name"]').extract() - #i['description'] = response.xpath('//div[@id="description"]').extract() - return i + item = {} + #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() + #item['name'] = response.xpath('//div[@id="name"]').get() + #item['description'] = response.xpath('//div[@id="description"]').get() + return item diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 7c2ff8850..863c9772f 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -10,8 +10,8 @@ class $classname(XMLFeedSpider): itertag = 'item' # change it accordingly def parse_node(self, response, selector): - i = {} - #i['url'] = selector.select('url').extract() - #i['name'] = selector.select('name').extract() - #i['description'] = selector.select('description').extract() - return i + item = {} + #item['url'] = selector.select('url').get() + #item['name'] = selector.select('name').get() + #item['description'] = selector.select('description').get() + return item From 460f0f045141f73dcc6d7a2309b7e2f0c2a492b4 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:20:32 +0500 Subject: [PATCH 0175/1853] [backwards incompatible] switch ItemLoader from .extract to .getall. This change is backwards incompatible if ItemLoader is used with a custom Selector subclass which overrides .extract without overriding .getall. --- scrapy/loader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index e73413318..a7c75a46a 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -181,7 +181,7 @@ class ItemLoader(object): def _get_xpathvalues(self, xpaths, **kw): self._check_selector_method() xpaths = arg_to_iter(xpaths) - return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths) + return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths) def add_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) @@ -198,6 +198,6 @@ class ItemLoader(object): def _get_cssvalues(self, csss, **kw): self._check_selector_method() csss = arg_to_iter(csss) - return flatten(self.selector.css(css).extract() for css in csss) + return flatten(self.selector.css(css).getall() for css in csss) XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) From 12e42bbe06bea5a0f86e95630b184e86b41c95c7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:20:55 +0500 Subject: [PATCH 0176/1853] switch SgmlLinkExtractor to .getall --- scrapy/linkextractors/sgml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index f4ca4262a..5fa6b771c 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): base_url = get_base_url(response) body = u''.join(f for x in self.restrict_xpaths - for f in response.xpath(x).extract() + for f in response.xpath(x).getall() ).encode(response.encoding, errors='xmlcharrefreplace') else: body = response.body From afce9716fabff6169dc9cc146a2589ebf4c1ed13 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:25:43 +0500 Subject: [PATCH 0177/1853] DOC mention .attrib in the tutorial --- docs/intro/tutorial.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 92d1065af..07fb4807f 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -520,6 +520,12 @@ like this:: >>> response.css('li.next a::attr(href)').get() '/page/2/' +There is also an ``attrib`` property available +(see :ref:`selecting-attributes` for more):: + + >>> response.css('li.next a').attrib['href'] + '/page/2' + Let's see now our spider modified to recursively follow the link to the next page, extracting data from it:: From bdcc045f62a34d1655a4ead4e0b38d6fe97a08fc Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Wed, 22 Aug 2018 04:27:21 +0500 Subject: [PATCH 0178/1853] DOC switch from .extract to get/getall API in docs Also, response.urljoin is added in a few places, for robustness. --- docs/intro/tutorial.rst | 22 +++++++++++----------- docs/topics/shell.rst | 4 ++-- docs/topics/spiders.rst | 14 +++++++------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 07fb4807f..46e84b21c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -254,7 +254,7 @@ data. To extract the text from the title above, you can do:: - >>> response.css('title::text').extract() + >>> response.css('title::text').getall() ['Quotes to Scrape'] There are two things to note here: one is that we've added ``::text`` to the @@ -262,12 +262,12 @@ CSS query, to mean we want to select only the text elements directly inside ``<title>`` element. If we don't specify ``::text``, we'd get the full title element, including its tags:: - >>> response.css('title').extract() + >>> response.css('title').getall() ['<title>Quotes to Scrape'] -The other thing is that the result of calling ``.extract()`` is a list, because -we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When -you know you just want the first result, as in this case, you can do:: +The other thing is that the result of calling ``.getall()`` is a list: it is +possible that a selector returns more than one result, so we extract them all. +When you know you just want the first result, as in this case, you can do:: >>> response.css('title::text').get() 'Quotes to Scrape' @@ -392,10 +392,10 @@ using the ``quote`` object we just created:: >>> author 'Albert Einstein' -Given that the tags are a list of strings, we can use the ``.extract()`` method +Given that the tags are a list of strings, we can use the ``.getall()`` method to get all of them:: - >>> tags = quote.css("div.tags a.tag::text").extract() + >>> tags = quote.css("div.tags a.tag::text").getall() >>> tags ['change', 'deep-thoughts', 'thinking', 'world'] @@ -405,7 +405,7 @@ quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): ... text = quote.css("span.text::text").get() ... author = quote.css("small.author::text").get() - ... tags = quote.css("div.tags a.tag::text").extract() + ... tags = quote.css("div.tags a.tag::text").getall() ... print(dict(text=text, author=author, tags=tags)) {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} @@ -438,7 +438,7 @@ in the callback, as you can see below:: yield { 'text': quote.css('span.text::text').get(), 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'tags': quote.css('div.tags a.tag::text').getall(), } If you run this spider, it will output the extracted data with the log:: @@ -543,7 +543,7 @@ page, extracting data from it:: yield { 'text': quote.css('span.text::text').get(), 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'tags': quote.css('div.tags a.tag::text').getall(), } next_page = response.css('li.next a::attr(href)').get() @@ -594,7 +594,7 @@ As a shortcut for creating Request objects you can use yield { 'text': quote.css('span.text::text').get(), 'author': quote.css('span small::text').get(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'tags': quote.css('div.tags a.tag::text').getall(), } next_page = response.css('li.next a::attr(href)').get() diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 9de6abef7..68a0b19b5 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -184,8 +184,8 @@ After that, we can start playing with the objects:: >>> fetch("https://reddit.com") - >>> response.xpath('//title/text()').extract() - ['reddit: the front page of the internet'] + >>> response.xpath('//title/text()').get() + 'reddit: the front page of the internet' >>> request = request.replace(method="POST") diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4505b7315..a08dc30f2 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback:: ] def parse(self, response): - for h3 in response.xpath('//h3').extract(): + for h3 in response.xpath('//h3').getall(): yield {"title": h3} - for url in response.xpath('//a/@href').extract(): - yield scrapy.Request(url, callback=self.parse) + for href in response.xpath('//a/@href').getall(): + yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; to give data more structure you can use :ref:`topics-items`:: @@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`:: yield scrapy.Request('http://www.example.com/3.html', self.parse) def parse(self, response): - for h3 in response.xpath('//h3').extract(): + for h3 in response.xpath('//h3').getall(): yield MyItem(title=h3) - for url in response.xpath('//a/@href').extract(): - yield scrapy.Request(url, callback=self.parse) + for href in response.xpath('//a/@href').getall(): + yield scrapy.Request(response.urljoin(href), self.parse) .. _spiderargs: @@ -545,7 +545,7 @@ These spiders are pretty easy to use, let's have a look at one example:: itertag = 'item' def parse_node(self, response, node): - self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract())) + self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall())) item = TestItem() item['id'] = node.xpath('@id').get() From 7fdfdb7fa244ed44274aaa9750efec1c333eedd1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 22 Aug 2018 17:34:44 +0500 Subject: [PATCH 0179/1853] DOC reorganize selectors tutorial, port more topics from parsel docs, adjust wording in the introduction --- docs/topics/selectors.rst | 514 +++++++++++++++++++++----------------- 1 file changed, 281 insertions(+), 233 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index cf1b0ba22..2db982b37 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -25,17 +25,14 @@ either by `XPath`_ or `CSS`_ expressions. used with HTML. `CSS`_ is a language for applying styles to HTML documents. It defines selectors to associate those styles with specific HTML elements. -Scrapy selectors are powered by `parsel`_ library, which uses `lxml`_ library -under the hood. It means Scrapy selectors are very similar in speed and -parsing accuracy to lxml. +.. note:: + Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of + this wrapper is to provide better integration with Scrapy Response objects. -This page explains how selectors work and describes their API which is very -small and simple, unlike the `lxml`_ API which is much bigger because the -`lxml`_ library can be used for many other tasks, besides selecting markup -documents. - -For a complete reference of the selectors API see -:ref:`Selector reference ` + `parsel`_ is a stand-alone web scraping library which can be used without + Scrapy. It uses `lxml`_ library under the hood, and implements an + easy API on top of lxml API. It means Scrapy selectors are very similar + in speed and parsing accuracy to lxml. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ @@ -73,9 +70,8 @@ Constructing from response:: >>> Selector(response=response).xpath('//span/text()').get() 'good' -For convenience, response objects expose a selector on `.selector` attribute, -it's totally OK to use this shortcut when possible. By using it you can -ensure the response body is parsed only once:: +For convenience, response objects expose a selector on `.selector` attribute. +By using it you can ensure the response body is parsed only once:: >>> response.selector.xpath('//span/text()').get() 'good' @@ -88,8 +84,10 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:: >>> response.css('span::text').get() 'good' -Usually there is no need to construct Scrapy selectors manually because of -these shortcuts. +Usually there is no need to construct Scrapy selectors manually: +``response`` object is available in Spider callbacks, so in most cases +it is more convenient to use ``response.css()`` and ``response.xpath()`` +shortcuts. Using selectors --------------- @@ -392,6 +390,71 @@ Use it to extract just the first matching string:: >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') 'My image 1' +.. _old-extraction-api: + +extract() and extract_first() +----------------------------- + +If you're a long-time Scrapy user, you're probably familiar +with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts +and tutorials are using them as well. These methods are still supported +by Scrapy, there are **no plans** to deprecate them. + +However, Scrapy usage docs are now written using ``.get()`` and +``.getall()`` methods. We feel that these new methods result in a more concise +and readable code. + +The following examples show how these methods map to each other. + +1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:: + + >>> response.css('a::attr(href)').get() + 'image1.html' + >>> response.css('a::attr(href)').extract_first() + 'image1.html' + +2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + >>> response.css('a::attr(href)').extract() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +2. ``Selector.get()`` is the same as ``Selector.extract()``:: + + >>> response.css('a::attr(href)')[0].get() + 'image1.html' + >>> response.css('a::attr(href)')[0].extract() + 'image1.html' + +4. For consistency, there is also ``Selector.getall()``, which returns a list:: + + >>> response.css('a::attr(href)')[0].getall() + ['image1.html'] + +So, the main difference is that output of ``.get()`` and ``.getall()`` methods +is more predictable: ``.get()`` always returns a single result, ``.getall()`` +always returns a list of all extracted results. With ``.extract()`` method +it was not always obvious if a result is a list or not; to get a single +result either ``.extract()`` or ``.extract_first()`` should be called. + + +.. _topics-selectors-xpaths: + +Working with XPaths +=================== + +Here are some tips which may help you to use XPath with Scrapy selectors +effectively. If you are not much familiar with XPath yet, +you may want to take a look first at this `XPath tutorial`_. + +.. note:: + Some of the tips are based on `this post from ScrapingHub's blog`_. + +.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html +.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ + + .. _topics-selectors-relative-xpaths: Working with relative XPaths @@ -428,6 +491,114 @@ XPath specification. .. _Location Paths: https://www.w3.org/TR/xpath#location-paths +When querying by class, consider using CSS +------------------------------------------ + +Because an element can contain multiple CSS classes, the XPath way to select elements +by class is the rather verbose:: + + *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')] + +If you use ``@class='someclass'`` you may end up missing elements that have +other classes, and if you just use ``contains(@class, 'someclass')`` to make up +for that you may end up with more elements that you want, if they have a different +class name that shares the string ``someclass``. + +As it turns out, Scrapy selectors allow you to chain selectors, so most of the time +you can just select by class using CSS and then switch to XPath when needed:: + + >>> from scrapy import Selector + >>> sel = Selector(text='
        ') + >>> sel.css('.shout').xpath('./time/@datetime').getall() + ['2014-07-23 19:00'] + +This is cleaner than using the verbose XPath trick shown above. Just remember +to use the ``.`` in the XPath expressions that will follow. + +Beware of the difference between //node[1] and (//node)[1] +---------------------------------------------------------- + +``//node[1]`` selects all the nodes occurring first under their respective parents. + +``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them. + +Example:: + + >>> from scrapy import Selector + >>> sel = Selector(text=""" + ....:
          + ....:
        • 1
          • + ....:
        • 2
          • + ....:
        • 3
          • + ....:
        + ....:
          + ....:
        • 4
          • + ....:
        • 5
          • + ....:
        • 6
          • + ....:
        """) + >>> xp = lambda x: sel.xpath(x).getall() + +This gets all first ``
      • `` elements under whatever it is its parent:: + + >>> xp("//li[1]") + ['
      • 1
        • ', '
      • 4
        • '] + +And this gets the first ``
      • `` element in the whole document:: + + >>> xp("(//li)[1]") + ['
      • 1
        • '] + +This gets all first ``
      • `` elements under an ``
          `` parent:: + + >>> xp("//ul/li[1]") + ['
        • 1
          • ', '
        • 4
          • '] + +And this gets the first ``
        • `` element under an ``
            `` parent in the whole document:: + + >>> xp("(//ul/li)[1]") + ['
          • 1
            • '] + +Using text nodes in a condition +------------------------------- + +When you need to use the text content as argument to an `XPath string function`_, +avoid using ``.//text()`` and use just ``.`` instead. + +This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*. +And when a node-set is converted to a string, which happens when it is passed as argument to +a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only. + +Example:: + + >>> from scrapy import Selector + >>> sel = Selector(text='Click here to go to the Next Page') + +Converting a *node-set* to string:: + + >>> sel.xpath('//a//text()').getall() # take a peek at the node-set + ['Click here to go to the ', 'Next Page'] + >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string + ['Click here to go to the '] + +A *node* converted to a string, however, puts together the text of itself plus of all its descendants:: + + >>> sel.xpath("//a[1]").getall() # select the first node + ['Click here to go to the Next Page'] + >>> sel.xpath("string(//a[1])").getall() # convert it to string + ['Click here to go to the Next Page'] + +So, using the ``.//text()`` node-set won't select anything in this case:: + + >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall() + [] + +But using the ``.`` to mean the node, works:: + + >>> sel.xpath("//a[contains(., 'Next Page')]").getall() + ['Click here to go to the Next Page'] + +.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions + .. _topics-selectors-xpath-variables: Variables in XPath expressions @@ -461,6 +632,69 @@ on `XPath variables`_. .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions + +.. _removing-namespaces: + +Removing namespaces +------------------- + +When dealing with scraping projects, it is often quite convenient to get rid of +namespaces altogether and just work with element names, to write more +simple/convenient XPaths. You can use the +:meth:`Selector.remove_namespaces` method for that. + +Let's show an example that illustrates this with GitHub blog atom feed. + +.. highlight:: sh + +First, we open the shell with the url we want to scrape:: + + $ scrapy shell https://github.com/blog.atom + +.. highlight:: xml + +This is how the file starts:: + + + + tag:github.com,2008:/blog + ... + +You can see two namespace declarations: a default "http://www.w3.org/2005/Atom" +and another one using the "media:" prefix for "http://search.yahoo.com/mrss/". + +.. highlight:: python + +Once in the shell we can try selecting all ```` objects and see that it +doesn't work (because the Atom XML namespace is obfuscating those nodes):: + + >>> response.xpath("//link") + [] + +But once we call the :meth:`Selector.remove_namespaces` method, all +nodes can be accessed directly by their names:: + + >>> response.selector.remove_namespaces() + >>> response.xpath("//link") + [, + , + ... + +If you wonder why the namespace removal procedure isn't always called by default +instead of having to call it manually, this is because of two reasons, which, in order +of relevance, are: + +1. Removing namespaces requires to iterate and modify all nodes in the + document, which is a reasonably expensive operation to perform by default + for all documents crawled by Scrapy + +2. There could be some cases where using namespaces is actually required, in + case some element names clash between namespaces. These cases are very rare + though. + + Using EXSLT extensions ---------------------- @@ -606,174 +840,44 @@ inside another ``itemscope``. .. _regular expressions: http://exslt.org/regexp/index.html .. _set manipulation: http://exslt.org/set/index.html +Other XPath extensions +---------------------- -Some XPath tips ---------------- +Scrapy selectors also provide a sorely missed XPath extension function +``has-class`` that returns ``True`` for nodes that have all of the specified +HTML classes. -Here are some tips that you may find useful when using XPath -with Scrapy selectors, based on `this post from ScrapingHub's blog`_. -If you are not much familiar with XPath yet, -you may want to take a look first at this `XPath tutorial`_. +.. highlight:: html +For the following HTML:: -.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ +

            First

            +

            Second

            +

            Third

            +

            Fourth

            +.. highlight:: python -Using text nodes in a condition -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can use it like this:: -When you need to use the text content as argument to an `XPath string function`_, -avoid using ``.//text()`` and use just ``.`` instead. - -This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*. -And when a node-set is converted to a string, which happens when it is passed as argument to -a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only. - -Example:: - - >>> from scrapy import Selector - >>> sel = Selector(text='Click here to go to the Next Page') - -Converting a *node-set* to string:: - - >>> sel.xpath('//a//text()').getall() # take a peek at the node-set - ['Click here to go to the ', 'Next Page'] - >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string - ['Click here to go to the '] - -A *node* converted to a string, however, puts together the text of itself plus of all its descendants:: - - >>> sel.xpath("//a[1]").getall() # select the first node - ['Click here to go to the Next Page'] - >>> sel.xpath("string(//a[1])").getall() # convert it to string - ['Click here to go to the Next Page'] - -So, using the ``.//text()`` node-set won't select anything in this case:: - - >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall() + >>> response.xpath('//p[has-class("foo")]') + [, + ] + >>> response.xpath('//p[has-class("foo", "bar-baz")]') + [] + >>> response.xpath('//p[has-class("foo", "bar")]') [] -But using the ``.`` to mean the node, works:: +So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS +``p.foo.bar-baz``. Please note, that it is slower in most of the cases, +because it's a pure-Python function that's invoked for every node in question +whereas the CSS lookup is translated into XPath and thus runs more efficiently, +so performance-wise its uses are limited to situations that are not easily +described with CSS selectors. - >>> sel.xpath("//a[contains(., 'Next Page')]").getall() - ['Click here to go to the Next Page'] +Parsel also simplifies adding your own XPath extensions. -.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions - -Beware of the difference between //node[1] and (//node)[1] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``//node[1]`` selects all the nodes occurring first under their respective parents. - -``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them. - -Example:: - - >>> from scrapy import Selector - >>> sel = Selector(text=""" - ....:
              - ....:
            • 1
              • - ....:
            • 2
              • - ....:
            • 3
              • - ....:
            - ....:
              - ....:
            • 4
              • - ....:
            • 5
              • - ....:
            • 6
              • - ....:
            """) - >>> xp = lambda x: sel.xpath(x).getall() - -This gets all first ``
          • `` elements under whatever it is its parent:: - - >>> xp("//li[1]") - ['
          • 1
            • ', '
          • 4
            • '] - -And this gets the first ``
          • `` element in the whole document:: - - >>> xp("(//li)[1]") - ['
          • 1
            • '] - -This gets all first ``
          • `` elements under an ``
              `` parent:: - - >>> xp("//ul/li[1]") - ['
            • 1
              • ', '
            • 4
              • '] - -And this gets the first ``
            • `` element under an ``
                `` parent in the whole document:: - - >>> xp("(//ul/li)[1]") - ['
              • 1
                • '] - -When querying by class, consider using CSS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Because an element can contain multiple CSS classes, the XPath way to select elements -by class is the rather verbose:: - - *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')] - -If you use ``@class='someclass'`` you may end up missing elements that have -other classes, and if you just use ``contains(@class, 'someclass')`` to make up -for that you may end up with more elements that you want, if they have a different -class name that shares the string ``someclass``. - -As it turns out, Scrapy selectors allow you to chain selectors, so most of the time -you can just select by class using CSS and then switch to XPath when needed:: - - >>> from scrapy import Selector - >>> sel = Selector(text='
                ') - >>> sel.css('.shout').xpath('./time/@datetime').getall() - ['2014-07-23 19:00'] - -This is cleaner than using the verbose XPath trick shown above. Just remember -to use the ``.`` in the XPath expressions that will follow. - -.. _old-extraction-api: - -extract() and extract_first() ------------------------------ - -If you're a long-time Scrapy user, you're probably familiar -with ``.extract()`` and ``.extract_first()`` selector methods. These methods -are still supported by Scrapy, there are no plans to deprecate them. - -However, Scrapy usage docs are now written using ``.get()`` and -``.getall()`` methods. We feel that these new methods result in a more concise -and readable code. - -The following examples show how these methods map to each other. - -1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:: - - >>> response.css('a::attr(href)').get() - 'image1.html' - >>> response.css('a::attr(href)').extract_first() - 'image1.html' - -2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:: - - >>> response.css('a::attr(href)').getall() - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] - >>> response.css('a::attr(href)').extract() - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] - -2. ``Selector.get()`` is the same as ``Selector.extract()``:: - - >>> response.css('a::attr(href)')[0].get() - 'image1.html' - >>> response.css('a::attr(href)')[0].extract() - 'image1.html' - -4. For consistency, there is also ``Selector.getall()``, which returns a list:: - - >>> response.css('a::attr(href)')[0].getall() - ['image1.html'] - -So, the main difference is that output of ``.get()`` and ``.getall()`` methods -is more predictable: ``.get()`` always returns a single result, ``.getall()`` -always returns a list of all extracted results. With ``.extract()`` method -it was not always obvious if a result is a list or not; to get a single -result either ``.extract()`` or ``.extract_first()`` should be called. +.. autofunction:: parsel.xpathfuncs.set_xpathfunc .. _topics-selectors-ref: @@ -909,6 +1013,11 @@ SelectorList objects their results flattened, as a list of unicode strings. +.. _selector-examples: + +Examples +======== + .. _selector-examples-html: Selector examples on HTML response @@ -958,65 +1067,4 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object:: sel.register_namespace("g", "http://base.google.com/ns/1.0") sel.xpath("//g:price").getall() -.. _removing-namespaces: - -Removing namespaces -------------------- - -When dealing with scraping projects, it is often quite convenient to get rid of -namespaces altogether and just work with element names, to write more -simple/convenient XPaths. You can use the -:meth:`Selector.remove_namespaces` method for that. - -Let's show an example that illustrates this with GitHub blog atom feed. - -.. highlight:: sh - -First, we open the shell with the url we want to scrape:: - - $ scrapy shell https://github.com/blog.atom - -.. highlight:: xml - -This is how the file starts:: - - - - tag:github.com,2008:/blog - ... - -You can see two namespace declarations: a default "http://www.w3.org/2005/Atom" -and another one using the "media:" prefix for "http://search.yahoo.com/mrss/". - -.. highlight:: python - -Once in the shell we can try selecting all ```` objects and see that it -doesn't work (because the Atom XML namespace is obfuscating those nodes):: - - >>> response.xpath("//link") - [] - -But once we call the :meth:`Selector.remove_namespaces` method, all -nodes can be accessed directly by their names:: - - >>> response.selector.remove_namespaces() - >>> response.xpath("//link") - [, - , - ... - -If you wonder why the namespace removal procedure isn't always called by default -instead of having to call it manually, this is because of two reasons, which, in order -of relevance, are: - -1. Removing namespaces requires to iterate and modify all nodes in the - document, which is a reasonably expensive operation to perform by default - for all documents crawled by Scrapy - -2. There could be some cases where using namespaces is actually required, in - case some element names clash between namespaces. These cases are very rare - though. - .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799 From dc95ecbe25a5921f902fbccc93d50f7682cefda2 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Sep 2018 18:36:25 +0500 Subject: [PATCH 0180/1853] DOC use autodocs for selectors; document more methods and attributes; suggest get/getall --- docs/topics/selectors.rst | 114 +++++++++---------------------------- scrapy/selector/unified.py | 33 +++++++++++ 2 files changed, 60 insertions(+), 87 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 2db982b37..95d6a1a04 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -891,127 +891,67 @@ Built-in Selectors reference Selector objects ---------------- -.. class:: Selector(response=None, text=None, type=None) +.. autoclass:: Selector - An instance of :class:`Selector` is a wrapper over response to select - certain parts of its content. - - ``response`` is an :class:`~scrapy.http.HtmlResponse` or an - :class:`~scrapy.http.XmlResponse` object that will be used for selecting and - extracting data. - - ``text`` is a unicode string or utf-8 encoded text for cases when a - ``response`` isn't available. Using ``text`` and ``response`` together is - undefined behavior. - - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default). - - If ``type`` is ``None``, the selector automatically chooses the best type - based on ``response`` type (see below), or defaults to ``"html"`` in case it - is used together with ``text``. - - If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follows: - - * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type - * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type - * ``"html"`` for anything else - - Otherwise, if ``type`` is set, the selector type will be forced and no - detection will occur. - - .. method:: xpath(query) - - Find nodes matching the xpath ``query`` and return the result as a - :class:`SelectorList` instance with all elements flattened. List - elements implement :class:`Selector` interface too. - - ``query`` is a string containing the XPATH query to apply. + .. automethod:: xpath .. note:: For convenience, this method can be called as ``response.xpath()`` - .. method:: css(query) - - Apply the given CSS selector and return a :class:`SelectorList` instance. - - ``query`` is a string containing the CSS selector to apply. - - In the background, CSS queries are translated into XPath queries using - `cssselect`_ library and run ``.xpath()`` method. + .. automethod:: css .. note:: - For convenience this method can be called as ``response.css()`` + For convenience, this method can be called as ``response.css()`` - .. method:: extract() + .. automethod:: get - Serialize and return the matched nodes as a list of unicode strings. - Percent encoded content is unquoted. + See also: :ref:`old-extraction-api` - .. method:: re(regex) + .. autoattribute:: attrib - Apply the given regex and return a list of unicode strings with the - matches. + See also: :ref:`selecting-attributes`. - ``regex`` can be either a compiled regular expression or a string which - will be compiled to a regular expression using ``re.compile(regex)`` + .. automethod:: re - .. note:: + .. automethod:: re_first - Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``<`` and ``&``). + .. automethod:: register_namespace - .. method:: register_namespace(prefix, uri) + .. automethod:: remove_namespaces - Register the given namespace to be used in this :class:`Selector`. - Without registering namespaces you can't select or extract data from - non-standard namespaces. See examples below. + .. automethod:: __bool__ - .. method:: remove_namespaces() - - Remove all namespaces, allowing to traverse the document using - namespace-less xpaths. See example below. - - .. method:: __nonzero__() - - Returns ``True`` if there is any real content selected or ``False`` - otherwise. In other words, the boolean value of a :class:`Selector` is - given by the contents it selects. + .. automethod:: getall + This method is added to Selector for consistency; it is more useful + with SelectorList. See also: :ref:`old-extraction-api` SelectorList objects -------------------- -.. class:: SelectorList +.. autoclass:: SelectorList - The :class:`SelectorList` class is a subclass of the builtin ``list`` - class, which provides a few additional methods. + .. automethod:: xpath - .. method:: xpath(query) + .. automethod:: css - Call the ``.xpath()`` method for each element in this list and return - their results flattened as another :class:`SelectorList`. + .. automethod:: getall - ``query`` is the same argument as the one in :meth:`Selector.xpath` + See also: :ref:`old-extraction-api` - .. method:: css(query) + .. automethod:: get - Call the ``.css()`` method for each element in this list and return - their results flattened as another :class:`SelectorList`. + See also: :ref:`old-extraction-api` - ``query`` is the same argument as the one in :meth:`Selector.css` + .. automethod:: re - .. method:: extract() + .. automethod:: re_first - Call the ``.extract()`` method for each element in this list and return - their results flattened, as a list of unicode strings. - - .. method:: re() - - Call the ``.re()`` method for each element in this list and return - their results flattened, as a list of unicode strings. + .. autoattribute:: attrib + See also: :ref:`selecting-attributes`. .. _selector-examples: diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 64cb0232c..8f6cb1d79 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -27,6 +27,10 @@ def _response_from_text(text, st): class SelectorList(_ParselSelector.selectorlist_cls, object_ref): + """ + The :class:`SelectorList` class is a subclass of the builtin ``list`` + class, which provides a few additional methods. + """ @deprecated(use_instead='.extract()') def extract_unquoted(self): return [x.extract_unquoted() for x in self] @@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref): class Selector(_ParselSelector, object_ref): + """ + An instance of :class:`Selector` is a wrapper over response to select + certain parts of its content. + + ``response`` is an :class:`~scrapy.http.HtmlResponse` or an + :class:`~scrapy.http.XmlResponse` object that will be used for selecting + and extracting data. + + ``text`` is a unicode string or utf-8 encoded text for cases when a + ``response`` isn't available. Using ``text`` and ``response`` together is + undefined behavior. + + ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` + or ``None`` (default). + + If ``type`` is ``None``, the selector automatically chooses the best type + based on ``response`` type (see below), or defaults to ``"html"`` in case it + is used together with ``text``. + + If ``type`` is ``None`` and a ``response`` is passed, the selector type is + inferred from the response type as follows: + + * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type + * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"html"`` for anything else + + Otherwise, if ``type`` is set, the selector type will be forced and no + detection will occur. + """ __slots__ = ['response'] selectorlist_cls = SelectorList From 9db21e55028feec932136e74dad0c0f0dbe7f436 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 15 Sep 2018 02:43:37 +0500 Subject: [PATCH 0181/1853] DOC fix remove_namespaces example See https://github.com/scrapy/parsel/pull/119 --- docs/topics/selectors.rst | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 95d6a1a04..20c3fff3c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -643,27 +643,30 @@ namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the :meth:`Selector.remove_namespaces` method for that. -Let's show an example that illustrates this with GitHub blog atom feed. +Let's show an example that illustrates this with the Python Insider blog atom feed. .. highlight:: sh First, we open the shell with the url we want to scrape:: - $ scrapy shell https://github.com/blog.atom - -.. highlight:: xml + $ scrapy shell https://feeds.feedburner.com/PythonInsider This is how the file starts:: - - tag:github.com,2008:/blog + ... -You can see two namespace declarations: a default "http://www.w3.org/2005/Atom" -and another one using the "media:" prefix for "http://search.yahoo.com/mrss/". +You can see several namespace declarations including a default +"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for +"http://schemas.google.com/g/2005". .. highlight:: python @@ -678,8 +681,8 @@ nodes can be accessed directly by their names:: >>> response.selector.remove_namespaces() >>> response.xpath("//link") - [, - , + [, + , ... If you wonder why the namespace removal procedure isn't always called by default From 2c3b2158c99953823500f275ccc5206c11c9a811 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 18 Sep 2018 05:02:17 +0500 Subject: [PATCH 0182/1853] DOC address @stummjr's review comments * fixed several small issues * re-written "Creating Selectors" section * fixed remaining .extract usage in tests --- docs/intro/tutorial.rst | 6 ++-- docs/topics/selectors.rst | 52 +++++++++++++++++++---------------- tests/test_utils_iterators.py | 4 +-- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 46e84b21c..ad17ef096 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -285,9 +285,9 @@ There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail to be scraped, you can at least get **some** data. -Besides the :meth:`~scrapy.selector.Selector.extract` and +Besides the :meth:`~scrapy.selector.SelectorList.getall` and :meth:`~scrapy.selector.SelectorList.get` methods, you can also use -the :meth:`~scrapy.selector.Selector.re` method to extract using `regular +the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular expressions`_:: >>> response.css('title::text').re(r'Quotes.*') @@ -649,7 +649,7 @@ this time for scraping author information:: def parse_author(self, response): def extract_with_css(query): - return response.css(query).get().strip() + return response.css(query).get(default='').strip() yield { 'name': extract_with_css('h3.author-title::text'), diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 20c3fff3c..68913c697 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -50,28 +50,8 @@ Constructing selectors .. highlight:: python -Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class -constructed by passing **text** or :class:`~scrapy.http.TextResponse` -object. It automatically chooses the best parsing rules (XML vs HTML) based on -input type:: - - >>> from scrapy.selector import Selector - >>> from scrapy.http import HtmlResponse - -Constructing from text:: - - >>> body = 'good' - >>> Selector(text=body).xpath('//span/text()').get() - 'good' - -Constructing from response:: - - >>> response = HtmlResponse(url='http://example.com', body=body) - >>> Selector(response=response).xpath('//span/text()').get() - 'good' - -For convenience, response objects expose a selector on `.selector` attribute. -By using it you can ensure the response body is parsed only once:: +Response objects expose a :class:`~scrapy.selector.Selector` instance +on ``.selector`` attribute:: >>> response.selector.xpath('//span/text()').get() 'good' @@ -84,10 +64,34 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:: >>> response.css('span::text').get() 'good' +Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +constructed by passing either :class:`~scrapy.http.TextResponse` object or +markup as an unicode string (in ``text`` argument). Usually there is no need to construct Scrapy selectors manually: ``response`` object is available in Spider callbacks, so in most cases it is more convenient to use ``response.css()`` and ``response.xpath()`` -shortcuts. +shortcuts. By using ``response.selector`` or one of these shortcuts +you can also ensure the response body is parsed only once. + +But if required, it is possible to use ``Selector`` directly. +Constructing from text:: + + >>> from scrapy.selector import Selector + >>> body = 'good' + >>> Selector(text=body).xpath('//span/text()').get() + 'good' + +Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of +:class:`~scrapy.http.TextResponse` subclasses:: + + >>> from scrapy.selector import Selector + >>> from scrapy.http import HtmlResponse + >>> response = HtmlResponse(url='http://example.com', body=body) + >>> Selector(response=response).xpath('//span/text()').get() + 'good' + +``Selector`` automatically chooses the best parsing rules +(XML vs HTML) based on input type. Using selectors --------------- @@ -139,7 +143,7 @@ is returned. ``.getall()`` returns a list with all results. Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: - >>> selector.css('title::text').get() + >>> response.css('title::text').get() 'Example website' As you can see, ``.xpath()`` and ``.css()`` methods return a diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 00eb78068..2d845697e 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -32,8 +32,8 @@ class XmliterTestCase(unittest.TestCase): for x in self.xmliter(response, 'product'): attrs.append(( x.attrib['id'], - x.xpath("name/text()").extract(), - x.xpath("./type/text()").extract())) + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall())) self.assertEqual(attrs, [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])]) From ffbd33edac0367e9f975b9863e0c31e1c2b72ebc Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 18 Sep 2018 05:03:35 +0500 Subject: [PATCH 0183/1853] DOC mention gotcha with `foo::text` selector and empty `foo` elements also, move "Selecting attributes" reference closer to `a::atr(href)` example --- docs/topics/selectors.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 68913c697..9dced7473 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -279,6 +279,19 @@ Examples: 'Name: My image 5 ', '\n '] +* ``foo::text`` returns no results if ``foo`` element exists, but contains + no text (i.e. text is empty):: + + >>> response.css('img::text').getall() + [] + + This means ``.css('foo::text').get()`` could return None even if an element + exists. Use ``default=''`` if you always want a string:: + + >>> response.css('img::text').get() + >>> response.css('img::text').get(default='') + '' + * ``a::attr(href)`` selects the *href* attribute value of descendant links:: >>> response.css('a::attr(href)').getall() @@ -288,15 +301,14 @@ Examples: 'image4.html', 'image5.html'] +.. note:: + See also: :ref:`selecting-attributes`. + .. note:: You cannot chain these pseudo-elements. But in practice it would not make much sense: text nodes do not have attributes, and attribute values are string values already and do not have children nodes. -.. note:: - See also: :ref:`selecting-attributes`. - - .. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors .. _topics-selectors-nesting-selectors: From 37cfb49805c86168af7a831fc33ec4aeb83e53da Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 24 Sep 2018 16:42:49 -0300 Subject: [PATCH 0184/1853] Randomly generate telnet credentials by default --- scrapy/extensions/telnet.py | 25 +++++++++--- scrapy/settings/default_settings.py | 2 - tests/test_extension_telnet.py | 59 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/test_extension_telnet.py diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 93342f225..3d0afeffb 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -7,6 +7,8 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging import traceback +import binascii +import os from twisted.internet import protocol try: @@ -50,8 +52,21 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] - self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] + + username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) + if username: + self.username = username.encode('utf8') + else: + self.username = binascii.hexlify(os.urandom(8)) + + password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) + if password: + self.password = password.encode('utf8') + else: + self.password = binascii.hexlify(os.urandom(8)) + + logger.info('Telnet Username: %s' % self.username) + logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -74,8 +89,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username - and credentials.checkPassword(self.password)): + if not (credentials.username == self.username and + credentials.checkPassword(self.password)): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( @@ -104,7 +119,7 @@ class TelnetConsole(protocol.ServerFactory): 'p': pprint.pprint, 'prefs': print_live_refs, 'hpy': hpy, - 'help': "This is Scrapy telnet console. For more info see: " \ + 'help': "This is Scrapy telnet console. For more info see: " "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b7bc173c..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,8 +277,6 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' -TELNETCONSOLE_USERNAME = 'scrapy' -TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py new file mode 100644 index 000000000..ffea1c463 --- /dev/null +++ b/tests/test_extension_telnet.py @@ -0,0 +1,59 @@ +try: + import unittest.mock as mock +except ImportError: + import mock + +from twisted.trial import unittest +from twisted.conch.telnet import ITelnetProtocol +from twisted.cred import credentials +from twisted.internet import defer + +from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.utils.test import get_crawler + + +class TelnetExtensionTest(unittest.TestCase): + def _get_console_and_portal(self, settings=None): + crawler = get_crawler(settings_dict=settings) + console = TelnetConsole(crawler) + username = console.username + password = console.password + + def _get_telnet_vars(): + # This function has some side effects we don't need for this test + return {} + console._get_telnet_vars = _get_telnet_vars + + console.start_listening() + protocol = console.protocol() + portal = protocol.protocolArgs[0] + + return console, portal + + @defer.inlineCallbacks + def test_bad_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(b'username', b'password') + d = portal.login(creds, None, ITelnetProtocol) + yield self.assertFailure(d, ValueError) + console.stop_listening() + + @defer.inlineCallbacks + def test_good_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(console.username, console.password) + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() + + @defer.inlineCallbacks + def test_custom_credentials(self): + settings = { + 'TELNETCONSOLE_USERNAME': 'user', + 'TELNETCONSOLE_PASSWORD': 'pass', + } + console, portal = self._get_console_and_portal(settings=settings) + creds = credentials.UsernamePassword(b'user', b'pass') + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() From e57a629efc0846ed396247baf22d7846689b82e4 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 11:54:57 -0300 Subject: [PATCH 0185/1853] Generate only password, encode username/password only on login --- scrapy/extensions/telnet.py | 22 ++++++++-------------- scrapy/settings/default_settings.py | 2 ++ tests/test_extension_telnet.py | 5 ++++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3d0afeffb..6df435cef 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -53,20 +53,14 @@ class TelnetConsole(protocol.ServerFactory): self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) - if username: - self.username = username.encode('utf8') - else: - self.username = binascii.hexlify(os.urandom(8)) + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] - password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) - if password: - self.password = password.encode('utf8') - else: - self.password = binascii.hexlify(os.urandom(8)) + if not self.password: + self.password = binascii.hexlify(os.urandom(8)).decode('utf8') + logger.info('Telnet Username: %s', self.username) + logger.info('Telnet Password: %s', self.password) - logger.info('Telnet Username: %s' % self.username) - logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -89,8 +83,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username and - credentials.checkPassword(self.password)): + if not (credentials.username == self.username.encode('utf8') and + credentials.checkPassword(self.password.encode('utf8'))): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..3734a0a58 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = None SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index ffea1c463..487c7c29f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -41,7 +41,10 @@ class TelnetExtensionTest(unittest.TestCase): @defer.inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() - creds = credentials.UsernamePassword(console.username, console.password) + creds = credentials.UsernamePassword( + console.username.encode('utf8'), + console.password.encode('utf8') + ) d = portal.login(creds, None, ITelnetProtocol) yield d console.stop_listening() From 5f9931d2ada7a2a05df77b1c061eeb482fcda347 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:07:04 -0300 Subject: [PATCH 0186/1853] do not log username --- scrapy/extensions/telnet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 6df435cef..a3d55f3c6 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -58,7 +58,6 @@ class TelnetConsole(protocol.ServerFactory): if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode('utf8') - logger.info('Telnet Username: %s', self.username) logger.info('Telnet Password: %s', self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) From 441e1e750fe7ad970adafc4c1f42834f7db86d1d Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:28:34 -0300 Subject: [PATCH 0187/1853] Style changes --- scrapy/extensions/telnet.py | 3 +-- tests/test_extension_telnet.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index a3d55f3c6..dcf73eb88 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -52,7 +52,6 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] @@ -113,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 487c7c29f..4f389e5cb 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -12,17 +12,15 @@ from scrapy.extensions.telnet import TelnetConsole, logger from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) username = console.username password = console.password - def _get_telnet_vars(): - # This function has some side effects we don't need for this test - return {} - console._get_telnet_vars = _get_telnet_vars + # This function has some side effects we don't need for this test + console._get_telnet_vars = lambda: {} console.start_listening() protocol = console.protocol() From edaf74bfaeef7d995676ad8f6bfb8056a8e6966d Mon Sep 17 00:00:00 2001 From: jfflisikowski Date: Tue, 2 Oct 2018 19:48:48 +0200 Subject: [PATCH 0188/1853] Correct the unclear comments by adding <# < processing code not shown > --- docs/topics/debug.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d1991c02f..f93aa2c72 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -18,11 +18,13 @@ Consider the following scrapy spider below:: ) def parse(self, response): - # collect `item_urls` + # + # collect `item_urls` for item_url in item_urls: yield scrapy.Request(item_url, self.parse_item) def parse_item(self, response): + # item = MyItem() # populate `item` fields # and extract item_details_url From 58f5565357ed532970772cd55c2d17d1e00198a9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:23:12 -0300 Subject: [PATCH 0189/1853] Move MutableChain to scrapy.utils.python --- scrapy/core/spidermw.py | 21 +-------------------- scrapy/utils/python.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4268c91d6..d776430e5 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -11,32 +11,13 @@ from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list +from scrapy.utils.python import MutableChain def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') -class MutableChain: - """ - Thin wrapper around itertools.chain, allowing to add iterables "in-place" - """ - def __init__(self, *args): - self.data = chain(*args) - - def extend(self, *iterables): - self.data = chain(self.data, *iterables) - - def __iter__(self): - return self.data.__iter__() - - def __next__(self): # py3 - return self.data.__next__() - - def next(self): # py2 - return self.data.next() - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 732ca13a0..7971b4dde 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -9,6 +9,7 @@ import weakref import errno import six from functools import partial, wraps +from itertools import chain import sys from scrapy.utils.decorators import deprecated @@ -387,3 +388,23 @@ if hasattr(sys, "pypy_version_info"): else: def garbage_collect(): gc.collect() + + +class MutableChain(object): + """ + Thin wrapper around itertools.chain, allowing to add iterables "in-place" + """ + def __init__(self, *args): + self.data = chain(*args) + + def extend(self, *iterables): + self.data = chain(self.data, *iterables) + + def __iter__(self): + return self.data.__iter__() + + def __next__(self): # py3 + return self.data.__next__() + + def next(self): # py2 + return self.data.next() From a05eaeed73a469493e78b5a1c5f0b4de2adf41c2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:31:51 -0300 Subject: [PATCH 0190/1853] Simplify MutableChain --- scrapy/utils/python.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7971b4dde..1a6bab990 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -403,8 +403,7 @@ class MutableChain(object): def __iter__(self): return self.data.__iter__() - def __next__(self): # py3 - return self.data.__next__() + def __next__(self): + return next(self.data) - def next(self): # py2 - return self.data.next() + next = __next__ From 15f0a890ee9f059111333fdeb6c6c3b5a8dadc07 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:34:59 -0300 Subject: [PATCH 0191/1853] Assign processing methods to a variable before iterating --- scrapy/core/spidermw.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d776430e5..3fae770a9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -58,7 +58,8 @@ class SpiderMiddlewareManager(MiddlewareManager): # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index): + method_list = self.methods['process_spider_exception'][start_index:] + for method_index, method in enumerate(method_list, start=start_index): if method is None: continue result = method(response=response, exception=exception, spider=spider) @@ -88,7 +89,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recovered.extend(exception_result) - for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index): + method_list = self.methods['process_spider_output'][start_index:] + for method_index, method in enumerate(method_list, start=start_index): if method is None: continue # the following might fail directly if the output value is not a generator From e0360e5223b618934ee006b4c9ed63012e7e621f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:55:13 -0300 Subject: [PATCH 0192/1853] Add tests for MutableChain --- tests/test_utils_python.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index f6133657b..3e1148354 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,11 +9,23 @@ import six from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, - without_none_values) + without_none_values, MutableChain) __doctests__ = ['scrapy.utils.python'] +class MutableChainTest(unittest.TestCase): + def test_mutablechain(self): + m = MutableChain(range(2), [2, 3], (4, 5)) + m.extend(range(6, 7)) + m.extend([7, 8]) + m.extend([9, 10], (11, 12)) + self.assertEqual(next(m), 0) + self.assertEqual(m.next(), 1) + self.assertEqual(m.__next__(), 2) + self.assertEqual(list(m), list(range(3, 13))) + + class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') From 92b7955d75eba3ddad1e4815cb80cf60c7a9a7a9 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Tue, 16 Oct 2018 14:50:00 -0300 Subject: [PATCH 0194/1853] Add Telnet console authentication docs --- docs/topics/telnetconsole.rst | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index ce79c9f35..49c372598 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -26,8 +26,21 @@ The telnet console listens in the TCP port defined in the the console you need to type:: telnet localhost 6023 + Trying localhost... + Connected to localhost. + Escape character is '^]'. + Username: + Password: >>> - + +By default Username is ``scrapy`` and Password is autogenerated. The +autogenerated Password can be seen on scrapy logs like the example bellow:: + + 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 + +Default Username and Password can be overriden by the settings +:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD` + You need the telnet program which comes installed by default in Windows, and most Linux distros. @@ -160,3 +173,24 @@ Default: ``'127.0.0.1'`` The interface the telnet console should listen on + +.. setting:: TELNETCONSOLE_USERNAME + +TELNETCONSOLE_USERNAME +------------------ + +Default: ``'scrapy'`` + +The username used for the telnet console + + +.. setting:: TELNETCONSOLE_PASSWORD + +TELNETCONSOLE_PASSWORD +------------------ + +Default: ``None`` + +The password used for the telnet console, default behaviour is to have it +autogenerated + From 44f8e28b3c8608f65dbc7836b36bc231e38393b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 16 Oct 2018 19:53:20 -0300 Subject: [PATCH 0195/1853] Fix headings' underlines --- docs/topics/telnetconsole.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 49c372598..4db9cafb2 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -177,7 +177,7 @@ The interface the telnet console should listen on .. setting:: TELNETCONSOLE_USERNAME TELNETCONSOLE_USERNAME ------------------- +---------------------- Default: ``'scrapy'`` @@ -187,7 +187,7 @@ The username used for the telnet console .. setting:: TELNETCONSOLE_PASSWORD TELNETCONSOLE_PASSWORD ------------------- +---------------------- Default: ``None`` From c9b5bd6ad7728274b0f82fc3211c5ade5cd0d389 Mon Sep 17 00:00:00 2001 From: Immanuella Lim Date: Thu, 18 Oct 2018 02:22:32 +0800 Subject: [PATCH 0196/1853] Remove ad link Dive Into Python3 from tutorial docs --- docs/intro/tutorial.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ad17ef096..143e018ac 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -23,7 +23,7 @@ start by getting an idea of what the language is like, to get the most out of Scrapy. If you're already familiar with other languages, and want to learn Python -quickly, we recommend reading through `Dive Into Python 3`_. Alternatively, +quickly, we recommend reading through `Crash into Python`_. Alternatively, you can follow the `Python Tutorial`_. If you're new to programming and want to start with Python, the following books @@ -40,7 +40,7 @@ as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers -.. _Dive Into Python 3: http://www.diveintopython3.net +.. _Crash into Python: https://stephensugden.com/crash_into_python/ .. _Python Tutorial: https://docs.python.org/3/tutorial .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/ .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/ From f97e3e90f25c5077b47d9ec11a4cf84ea777227e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 29 Oct 2018 12:40:20 -0300 Subject: [PATCH 0197/1853] Use collections.deque instead of list to store methods --- scrapy/core/downloader/middleware.py | 4 ++-- scrapy/core/spidermw.py | 6 +++--- scrapy/middleware.py | 6 +++--- tests/test_middleware.py | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index c3b23e284..f5e2fca63 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_request'): self.methods['process_request'].append(mw.process_request) if hasattr(mw, 'process_response'): - self.methods['process_response'].insert(0, mw.process_response) + self.methods['process_response'].appendleft(mw.process_response) if hasattr(mw, 'process_exception'): - self.methods['process_exception'].insert(0, mw.process_exception) + self.methods['process_exception'].appendleft(mw.process_exception) def download(self, download_func, request, spider): @defer.inlineCallbacks diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a206e4b0c..16b8435ab 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_spider_output'): - self.methods['process_spider_output'].insert(0, mw.process_spider_output) + self.methods['process_spider_output'].appendleft(mw.process_spider_output) if hasattr(mw, 'process_spider_exception'): - self.methods['process_spider_exception'].insert(0, mw.process_spider_exception) + self.methods['process_spider_exception'].appendleft(mw.process_spider_exception) if hasattr(mw, 'process_start_requests'): - self.methods['process_start_requests'].insert(0, mw.process_start_requests) + self.methods['process_start_requests'].appendleft(mw.process_start_requests) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f2240984c..1cfd8a782 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,4 +1,4 @@ -from collections import defaultdict +from collections import defaultdict, deque import logging import pprint @@ -16,7 +16,7 @@ class MiddlewareManager(object): def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(list) + self.methods = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -56,7 +56,7 @@ class MiddlewareManager(object): if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): - self.methods['close_spider'].insert(0, mw.close_spider) + self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname, obj, *args): return process_parallel(self.methods[methodname], obj, *args) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6d885330..aea0be825 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase): def test_init(self): m1, m2, m3 = M1(), M2(), M3() mwman = TestMiddlewareManager(m1, m2, m3) - self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider]) - self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider]) - self.assertEqual(mwman.methods['process'], [m1.process, m3.process]) + self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider]) + self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider]) + self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process]) def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) From 6c98010f110c432a2311c1aef1d463dc5a6ccba4 Mon Sep 17 00:00:00 2001 From: Immanuella Lim Date: Sun, 4 Nov 2018 16:04:45 +0800 Subject: [PATCH 0198/1853] Remove 'Dive into Python3' reference --- docs/intro/tutorial.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 143e018ac..41e61542a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -22,9 +22,7 @@ Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of Scrapy. -If you're already familiar with other languages, and want to learn Python -quickly, we recommend reading through `Crash into Python`_. Alternatively, -you can follow the `Python Tutorial`_. +If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -40,7 +38,6 @@ as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers -.. _Crash into Python: https://stephensugden.com/crash_into_python/ .. _Python Tutorial: https://docs.python.org/3/tutorial .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/ .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/ From 491929c212999aa816e561aeed19a902664d01e5 Mon Sep 17 00:00:00 2001 From: Todd Date: Fri, 16 Nov 2018 13:38:19 -0500 Subject: [PATCH 0199/1853] Include additional files in sdists In particular this includes files needed for running the tests, as well as the changelog. --- MANIFEST.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 94de4f3bf..ae7db51fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,13 +3,24 @@ include AUTHORS include INSTALL include LICENSE include MANIFEST.in +include NEWS + include scrapy/VERSION include scrapy/mime.types + +include codecov.yml +include conftest.py +include pytest.ini +include requirements-*.txt +include tox.ini + recursive-include scrapy/templates * recursive-include scrapy license.txt recursive-include docs * prune docs/build + recursive-include extras * recursive-include bin * recursive-include tests * + global-exclude __pycache__ *.py[cod] From 127bf499f1d6b4a924d87e39ff89b528586c78c7 Mon Sep 17 00:00:00 2001 From: Frederik Elwert Date: Fri, 16 Nov 2018 22:15:03 +0100 Subject: [PATCH 0200/1853] Add documentation to `scrapy shell` command. The special syntax required for local files (`./file.html`) is not documented as part of the `scrapy shell --help` output. This patch adds that. --- scrapy/commands/shell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 40a58d94a..e05084272 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -28,7 +28,8 @@ class Command(ScrapyCommand): return "Interactive scraping console" def long_desc(self): - return "Interactive console for scraping the given url" + return ("Interactive console for scraping the given url or file. " + "Use ./file.html syntax or full path for local file.") def add_options(self, parser): ScrapyCommand.add_options(self, parser) From a25cf5c82f99f7ae11346a2e565d6255835c3814 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 20 Nov 2018 16:13:09 +0000 Subject: [PATCH 0201/1853] function to get unique file queues for any type of base queue --- scrapy/core/queues.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scrapy/core/queues.py diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py new file mode 100644 index 000000000..96d582fc7 --- /dev/null +++ b/scrapy/core/queues.py @@ -0,0 +1,15 @@ +import uuid +import os.path + + +def unique_files_queue(queue_class): + + class UniqueFilesQueue(queue_class): + def __init__(self, path): + path = path + "-" + uuid.uuid4().hex + while os.path.exists(path): + path = path + "-" + uuid.uuid4().hex + + super().__init__(path) + + return UniqueFilesQueue From 1ce6662a9d7115348788972afce62a5c45199021 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sat, 24 Nov 2018 20:02:00 +0530 Subject: [PATCH 0202/1853] Implement Request subclass for json requests --- scrapy/http/__init__.py | 1 + scrapy/http/request/json_request.py | 28 +++++++++++++++ tests/test_http_request.py | 55 ++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 scrapy/http/request/json_request.py diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index f04a9d3e5..4b2f7b33f 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -10,6 +10,7 @@ from scrapy.http.headers import Headers from scrapy.http.request import Request from scrapy.http.request.form import FormRequest from scrapy.http.request.rpc import XmlRpcRequest +from scrapy.http.request.json_request import JSONRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py new file mode 100644 index 000000000..0fdd2ddf1 --- /dev/null +++ b/scrapy/http/request/json_request.py @@ -0,0 +1,28 @@ +""" +This module implements the JSONRequest class which is a more convenient class +(than Request) to generate JSON Requests. + +See documentation in docs/topics/request-response.rst +""" + +import json + +from scrapy.http.request import Request + + +class JSONRequest(Request): + def __init__(self, *args, **kwargs): + if 'method' not in kwargs: + kwargs['method'] = 'POST' + + data = kwargs.pop('data', {}) + kwargs['body'] = json.dumps(data) + super(JSONRequest, self).__init__(*args, **kwargs) + self.headers.setdefault(b'Content-Type', b'application/json') + + def replace(self, *args, **kwargs): + """ Create a new Request with the same attributes except for those + given new values. """ + + kwargs.pop('body', None) + return super(JSONRequest, self).replace(*args, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..3f2e4f521 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -2,6 +2,7 @@ import cgi import unittest import re +import json import six from six.moves import xmlrpc_client as xmlrpclib @@ -9,7 +10,7 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote if six.PY3: from urllib.parse import unquote_to_bytes -from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse +from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str @@ -1147,5 +1148,57 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=(u'pas£',), encoding='latin1') +class JSONRequestTest(RequestTest): + request_class = JSONRequest + default_method = 'POST' + default_headers = {b'Content-Type': [b'application/json']} + + def test_body(self): + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.body, '{}') + + r2 = self.request_class(url="http://www.example.com/", body=b"") + self.assertEqual(r2.body, '{}') + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.body, json.dumps(data)) + + r4 = self.request_class(url="http://www.example.com/", body='body1', data=data) + self.assertEqual(r3.body, json.dumps(data)) + + def test_replace(self): + """Test Request.replace() method""" + r1 = self.request_class("http://www.example.com") + hdrs = Headers(r1.headers) + hdrs[b'key'] = b'value' + r2 = r1.replace(body="New body", headers=hdrs) + + # body will not be replaced + self.assertEqual(r1.body, r2.body) + self.assertEqual(r1.url, r2.url) + self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) + + # Empty attributes (which may fail if not compared properly) + r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) + r4 = r3.replace(url="http://www.example.com/2", meta={}, dont_filter=False) + self.assertEqual(r4.url, "http://www.example.com/2") + self.assertEqual(r4.meta, {}) + assert r4.dont_filter is False + + data1 = { + 'name': 'value1', + } + data2 = { + 'name': 'value2', + } + r5 = self.request_class("http://www.example.com", data=data1) + r6 = r5.replace(url="http://www.example.com/2", data=data2) + self.assertNotEqual(r5.body, r6.body) + self.assertEqual((r5.body, r6.body), (json.dumps(data1), json.dumps(data2))) + + if __name__ == "__main__": unittest.main() From 1b2b8b4bf0c73b4ad143f943584545702d66cbb7 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Tue, 27 Nov 2018 08:57:44 +0530 Subject: [PATCH 0203/1853] fix tests under py3 --- tests/test_http_request.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 3f2e4f521..a2021bd65 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1155,19 +1155,19 @@ class JSONRequestTest(RequestTest): def test_body(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, '{}') + self.assertEqual(r1.body, b'{}') r2 = self.request_class(url="http://www.example.com/", body=b"") - self.assertEqual(r2.body, '{}') + self.assertEqual(r2.body, b'{}') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.body, json.dumps(data)) + self.assertEqual(r3.body, to_bytes(json.dumps(data))) r4 = self.request_class(url="http://www.example.com/", body='body1', data=data) - self.assertEqual(r3.body, json.dumps(data)) + self.assertEqual(r3.body, to_bytes(json.dumps(data))) def test_replace(self): """Test Request.replace() method""" @@ -1197,7 +1197,7 @@ class JSONRequestTest(RequestTest): r5 = self.request_class("http://www.example.com", data=data1) r6 = r5.replace(url="http://www.example.com/2", data=data2) self.assertNotEqual(r5.body, r6.body) - self.assertEqual((r5.body, r6.body), (json.dumps(data1), json.dumps(data2))) + self.assertEqual((r5.body, r6.body), (to_bytes(json.dumps(data1)), to_bytes(json.dumps(data2)))) if __name__ == "__main__": From 274b65dff4dc8b8300d872171679f173fbe0a746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 16:36:05 +0100 Subject: [PATCH 0204/1853] Add a troubleshooting section to the installation instructions Its initial content covers the workaround for #2473. --- docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4a9aa3cfb..daec7fcb7 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. -For more detailed and platform specifics instructions, read on. +For more detailed and platform specifics instructions, as well as +troubleshooting information, read on. Things that are good to know @@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency. To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +.. _intro-install-troubleshooting: + +Troubleshooting +=============== + +AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' +---------------------------------------------------------------- + +After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an +exception with the following traceback:: + + […] + File "[…]/site-packages/twisted/protocols/tls.py", line 63, in + from twisted.internet._sslverify import _setAcceptableProtocols + File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in + TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1, + AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' + +The reason you get this exception is that your system or virtual environment +has a version of pyOpenSSL that your version of Twisted does not support. + +To install a version of pyOpenSSL that your version of Twisted supports, +reinstall Twisted with the :code:`tls` extra option:: + + pip install twisted[tls] + +For details, see `Issue #2473 `_. + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ From 62f3349c1aee54599ab7ee8755d2b31090639105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 17:14:10 +0100 Subject: [PATCH 0205/1853] Document the SCRAPY_PROJECT environment variable Fixes #1109 --- docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ef9c45196..97f8311de 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`) -* ``SCRAPY_PROJECT`` +* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`) * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) .. _topics-project-structure: @@ -71,6 +71,33 @@ the project settings. Here is an example:: [settings] default = myproject.settings +.. _topics-project-envvar: + +Sharing the root directory between projects +=========================================== + +A project root directory, the one that contains the ``scrapy.cfg``, may be +shared by multiple Scrapy projects, each with its own settings module. + +In that case, you must define one or more aliases for those settings modules +under ``[settings]`` in your ``scrapy.cfg`` file:: + + [settings] + default = myproject1.settings + project1 = myproject1.settings + project2 = myproject2.settings + +By default, the ``scrapy`` command-line tool will use the ``default`` settings. +Use the ``SCRAPY_PROJECT`` environment variable to specify a different project +for ``scrapy`` to use:: + + $ scrapy settings --get BOT_NAME + Project 1 Bot + $ export SCRAPY_PROJECT=project2 + $ scrapy settings --get BOT_NAME + Project 2 Bot + + Using the ``scrapy`` tool ========================= From d7c8eee2fc918d07feb708c41089da21b2b9aea5 Mon Sep 17 00:00:00 2001 From: fpghost Date: Tue, 4 Dec 2018 10:57:51 +0100 Subject: [PATCH 0206/1853] the strip() isnt needed --- scrapy/downloadermiddlewares/httpproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 1dd47359f..2c35d1b90 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -30,7 +30,7 @@ class HttpProxyMiddleware(object): user_pass = to_bytes( '%s:%s' % (unquote(username), unquote(password)), encoding=self.auth_encoding) - return base64.b64encode(user_pass).strip() + return base64.b64encode(user_pass) def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) From cd619c1d4f3810c96af0ff5c5735c1856dfac95a Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sat, 8 Dec 2018 22:10:45 +0530 Subject: [PATCH 0207/1853] removed overriden replace method --- scrapy/http/request/json_request.py | 20 +++++------ tests/test_http_request.py | 51 ++++++++--------------------- 2 files changed, 21 insertions(+), 50 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 0fdd2ddf1..03a0ab061 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -12,17 +12,13 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): - if 'method' not in kwargs: - kwargs['method'] = 'POST' + data = kwargs.pop('data', None) + if data: + kwargs['body'] = json.dumps(data) + + if 'method' not in kwargs: + kwargs['method'] = 'POST' - data = kwargs.pop('data', {}) - kwargs['body'] = json.dumps(data) super(JSONRequest, self).__init__(*args, **kwargs) - self.headers.setdefault(b'Content-Type', b'application/json') - - def replace(self, *args, **kwargs): - """ Create a new Request with the same attributes except for those - given new values. """ - - kwargs.pop('body', None) - return super(JSONRequest, self).replace(*args, **kwargs) + self.headers.setdefault('Content-Type', 'application/json') + self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a2021bd65..793a583bc 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1150,54 +1150,29 @@ class XmlRpcRequestTest(RequestTest): class JSONRequestTest(RequestTest): request_class = JSONRequest - default_method = 'POST' - default_headers = {b'Content-Type': [b'application/json']} + default_method = 'GET' + default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} - def test_body(self): + def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b'{}') + self.assertEqual(r1.body, b'') + self.assertEqual(r1.method, 'GET') - r2 = self.request_class(url="http://www.example.com/", body=b"") - self.assertEqual(r2.body, b'{}') + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.body, body) + self.assertEqual(r2.method, 'GET') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) + self.assertEqual(r3.method, 'POST') - r4 = self.request_class(url="http://www.example.com/", body='body1', data=data) - self.assertEqual(r3.body, to_bytes(json.dumps(data))) - - def test_replace(self): - """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com") - hdrs = Headers(r1.headers) - hdrs[b'key'] = b'value' - r2 = r1.replace(body="New body", headers=hdrs) - - # body will not be replaced - self.assertEqual(r1.body, r2.body) - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) - - # Empty attributes (which may fail if not compared properly) - r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) - r4 = r3.replace(url="http://www.example.com/2", meta={}, dont_filter=False) - self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.meta, {}) - assert r4.dont_filter is False - - data1 = { - 'name': 'value1', - } - data2 = { - 'name': 'value2', - } - r5 = self.request_class("http://www.example.com", data=data1) - r6 = r5.replace(url="http://www.example.com/2", data=data2) - self.assertNotEqual(r5.body, r6.body) - self.assertEqual((r5.body, r6.body), (to_bytes(json.dumps(data1)), to_bytes(json.dumps(data2)))) + r4 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r4.body, to_bytes(json.dumps(data))) + self.assertEqual(r4.method, 'POST') if __name__ == "__main__": From c347acbff6545c428aa2c965cd03f03db6bae1bf Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sun, 9 Dec 2018 11:27:09 +0530 Subject: [PATCH 0208/1853] warning if body and data are provided --- scrapy/http/request/json_request.py | 7 ++++++- tests/test_http_request.py | 25 ++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 03a0ab061..3b791eda3 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -6,14 +6,19 @@ See documentation in docs/topics/request-response.rst """ import json +import warnings from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): + body_passed = 'body' in kwargs data = kwargs.pop('data', None) - if data: + if body_passed and data: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data: kwargs['body'] = json.dumps(data) if 'method' not in kwargs: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 793a583bc..e5a85e6fc 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import cgi import unittest import re import json +import warnings import six from six.moves import xmlrpc_client as xmlrpclib @@ -1153,6 +1154,10 @@ class JSONRequestTest(RequestTest): default_method = 'GET' default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']} + def setUp(self): + warnings.simplefilter("always") + super(JSONRequestTest, self).setUp() + def test_data(self): r1 = self.request_class(url="http://www.example.com/") self.assertEqual(r1.body, b'') @@ -1170,9 +1175,23 @@ class JSONRequestTest(RequestTest): self.assertEqual(r3.body, to_bytes(json.dumps(data))) self.assertEqual(r3.method, 'POST') - r4 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r4.body, to_bytes(json.dumps(data))) - self.assertEqual(r4.method, 'POST') + with warnings.catch_warnings(record=True) as _warnings: + r4 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r4.body, body) + self.assertEqual(r4.method, 'GET') + self.assertEqual(len(_warnings), 1) + self.assertIn('data will be ignored', str(_warnings[0].message)) + + with warnings.catch_warnings(record=True) as _warnings: + r5 = self.request_class(url="http://www.example.com/", body=b'', data=data) + self.assertEqual(r5.body, b'') + self.assertEqual(r5.method, 'GET') + self.assertEqual(len(_warnings), 1) + self.assertIn('data will be ignored', str(_warnings[0].message)) + + def tearDown(self): + warnings.resetwarnings() + super(JSONRequestTest, self).tearDown() if __name__ == "__main__": From 3c981bf204c739fa77e205b9747d2aff446c99d5 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Sun, 9 Dec 2018 12:56:12 +0530 Subject: [PATCH 0209/1853] add documentation --- docs/topics/request-response.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e29914dbf..d957915e7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -508,6 +508,38 @@ method for this job. Here's an example spider which uses it:: # continue scraping with authenticated session... +JSONRequest +----------- + +The JSONRequest class extends the base :class:`Request` class with functionality for +dealing with JSON requests. + +.. class:: JSONRequest(url, [data, ...]) + + The :class:`JSONRequest` class adds a new argument to the constructor called data. The + remaining arguments are the same as for the :class:`Request` class and are + not documented here. + + Using the :class:`JSONRequest` will set the `Content-Type` header to `application/json` + and `Accept` header to `application/json, text/javascript, */*; q=0.01` + + :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. + if :attr:`Request.body` argument is provided this parameter will be ignored. + if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + set to POST automatically. + :type data: JSON serializable object + +JSONRequest usage example +------------------------- + +Sending a JSON POST request with a JSON payload:: + + data = { + 'name1': 'value1', + 'name2': 'value2', + } + yield JSONRequest(url='http://www.example.com/post/action', data=data) + Response objects ================ From 4d48759978ac2405bc2cb30f84af948693e4cad3 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 10 Dec 2018 14:44:15 +0800 Subject: [PATCH 0210/1853] remove "sudo: false" now that travis no longer supports it https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4218d13bf..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: python -sudo: false branches: only: - master From ecda69130e97629b15d3b09b1e588cb6777ee94d Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 10 Dec 2018 22:34:49 +0530 Subject: [PATCH 0211/1853] allow to send empty data values and docs changes --- docs/topics/request-response.rst | 6 +++--- scrapy/http/request/json_request.py | 8 +++++--- tests/test_http_request.py | 27 +++++++++++++++++++++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d957915e7..02b853fc0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -520,13 +520,13 @@ dealing with JSON requests. remaining arguments are the same as for the :class:`Request` class and are not documented here. - Using the :class:`JSONRequest` will set the `Content-Type` header to `application/json` - and `Accept` header to `application/json, text/javascript, */*; q=0.01` + Using the :class:`JSONRequest` will set the ``Content-Type`` header to ``application/json`` + and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01`` :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be - set to POST automatically. + set to ``'POST'`` automatically. :type data: JSON serializable object JSONRequest usage example diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 3b791eda3..593dfdcb0 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -13,12 +13,14 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): - body_passed = 'body' in kwargs + body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) - if body_passed and data: + data_passed = data is not None + + if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data: + elif not body_passed and data_passed: kwargs['body'] = json.dumps(data) if 'method' not in kwargs: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e5a85e6fc..5eb655c12 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1175,20 +1175,35 @@ class JSONRequestTest(RequestTest): self.assertEqual(r3.body, to_bytes(json.dumps(data))) self.assertEqual(r3.method, 'POST') + r4 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r4.body, to_bytes(json.dumps([]))) + self.assertEqual(r4.method, 'POST') + with warnings.catch_warnings(record=True) as _warnings: - r4 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r4.body, body) - self.assertEqual(r4.method, 'GET') + r5 = self.request_class(url="http://www.example.com/", body=body, data=data) + self.assertEqual(r5.body, body) + self.assertEqual(r5.method, 'GET') self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) with warnings.catch_warnings(record=True) as _warnings: - r5 = self.request_class(url="http://www.example.com/", body=b'', data=data) - self.assertEqual(r5.body, b'') - self.assertEqual(r5.method, 'GET') + r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) + self.assertEqual(r6.body, b'') + self.assertEqual(r6.method, 'GET') self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + with warnings.catch_warnings(record=True) as _warnings: + r7 = self.request_class(url="http://www.example.com/", body=None, data=data) + self.assertEqual(r7.body, to_bytes(json.dumps(data))) + self.assertEqual(r7.method, 'POST') + self.assertEqual(len(_warnings), 0) + + with warnings.catch_warnings(record=True) as _warnings: + r8 = self.request_class(url="http://www.example.com/", body=None, data=None) + self.assertEqual(r8.method, 'GET') + self.assertEqual(len(_warnings), 0) + def tearDown(self): warnings.resetwarnings() super(JSONRequestTest, self).tearDown() From cd9d8e28cdf49ff63e1b3f9126e6651fcd77e0fa Mon Sep 17 00:00:00 2001 From: hsiao yi Date: Tue, 11 Dec 2018 19:21:07 +0800 Subject: [PATCH 0212/1853] unify the quote style --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 9d7c94d39..8b2fef065 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -26,7 +26,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): - name = "quotes" + name = 'quotes' start_urls = [ 'http://quotes.toscrape.com/tag/humor/', ] From 71ef321b68d2fd202de145d0c580387ee59cd2e2 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Wed, 12 Dec 2018 11:12:48 +0530 Subject: [PATCH 0213/1853] sort_keys while serializing to json --- scrapy/http/request/json_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 593dfdcb0..afc4356a3 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -21,7 +21,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = json.dumps(data) + kwargs['body'] = json.dumps(data, sort_keys=True) if 'method' not in kwargs: kwargs['method'] = 'POST' From 8f1507a4a5de2ed55cb0fda198265845a047fedb Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 17 Dec 2018 23:14:06 +0530 Subject: [PATCH 0214/1853] dumps_kwargs --- docs/topics/request-response.rst | 10 ++- scrapy/http/request/json_request.py | 21 ++++- tests/test_http_request.py | 114 +++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 7 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 02b853fc0..4e6f00bb0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -514,9 +514,9 @@ JSONRequest The JSONRequest class extends the base :class:`Request` class with functionality for dealing with JSON requests. -.. class:: JSONRequest(url, [data, ...]) +.. class:: JSONRequest(url, [... data]) - The :class:`JSONRequest` class adds a new argument to the constructor called data. The + The :class:`JSONRequest` class adds two new argument to the constructor. The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -529,6 +529,12 @@ dealing with JSON requests. set to ``'POST'`` automatically. :type data: JSON serializable object + :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data + into JSON format. + :type dumps_kwargs: dict + +.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps + JSONRequest usage example ------------------------- diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index afc4356a3..7499610b9 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -13,6 +13,7 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): + dumps_kwargs = kwargs.pop('dumps_kwargs', {}) body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None @@ -21,7 +22,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = json.dumps(data, sort_keys=True) + kwargs['body'] = self.dump(data, **dumps_kwargs) if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -29,3 +30,21 @@ class JSONRequest(Request): super(JSONRequest, self).__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + self._dumps_kwargs = dumps_kwargs + + def replace(self, *args, **kwargs): + body_passed = kwargs.get('body', None) is not None + data = kwargs.pop('data', None) + data_passed = data is not None + + if body_passed and data_passed: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data_passed: + kwargs['body'] = self.dump(data, **self._dumps_kwargs) + + return super(JSONRequest, self).replace(*args, **kwargs) + + def dump(self, data, **kwargs): + """Convert to JSON """ + return json.dumps(data, sort_keys=True, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5eb655c12..6dcfa25da 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -14,6 +14,8 @@ if six.PY3: from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str +from tests import mock + class RequestTest(unittest.TestCase): @@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest): def test_data(self): r1 = self.request_class(url="http://www.example.com/") self.assertEqual(r1.body, b'') - self.assertEqual(r1.method, 'GET') body = b'body' r2 = self.request_class(url="http://www.example.com/", body=body) self.assertEqual(r2.body, body) - self.assertEqual(r2.method, 'GET') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) - self.assertEqual(r3.method, 'POST') + # empty data r4 = self.request_class(url="http://www.example.com/", data=[]) self.assertEqual(r4.body, to_bytes(json.dumps([]))) - self.assertEqual(r4.method, 'POST') + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.method, 'GET') + + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.method, 'GET') + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.method, 'POST') + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method='GET') + self.assertEqual(r4.method, 'GET') + + r5 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r5.method, 'POST') + + def test_body_data(self): + """ passing both body and data should result a warning """ + body = b'body' + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) self.assertEqual(r5.body, body) @@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest): self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + def test_empty_body_data(self): + """ passing any body value and data should result a warning """ + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) self.assertEqual(r6.body, b'') @@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest): self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + def test_body_none_data(self): + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) self.assertEqual(r7.body, to_bytes(json.dumps(data))) self.assertEqual(r7.method, 'POST') self.assertEqual(len(_warnings), 0) + def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) self.assertEqual(r8.method, 'GET') self.assertEqual(len(_warnings), 0) + def test_dumps_sort_keys(self): + """ Test that sort_keys=True is passed to json.dumps by default """ + data = { + 'name': 'value', + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_dumps_kwargs(self): + """ Test that dumps_kwargs are passed to json.dumps """ + data = { + 'name': 'value', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + + def test_replace_data(self): + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + + def test_replace_sort_keys(self): + """ Test that replace provides sort_keys=True to json.dumps """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_replace_dumps_kwargs(self): + """ Test that dumps_kwargs are provided json.dumps when replace is called """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + def tearDown(self): warnings.resetwarnings() super(JSONRequestTest, self).tearDown() From 12ad06b7ac57dd022a4add16259ee8fd64d5ede2 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 17 Dec 2018 23:17:13 +0530 Subject: [PATCH 0215/1853] docs change --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4e6f00bb0..6758269b1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -529,8 +529,8 @@ dealing with JSON requests. set to ``'POST'`` automatically. :type data: JSON serializable object - :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data - into JSON format. + :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize + data into JSON format. :type dumps_kwargs: dict .. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps From 24acc50d1894b6566e427f1dfea14e2aa647077e Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Tue, 18 Dec 2018 23:16:14 +0530 Subject: [PATCH 0216/1853] dumps_kwargs parameter in docs --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6758269b1..37b73edd1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -514,7 +514,7 @@ JSONRequest The JSONRequest class extends the base :class:`Request` class with functionality for dealing with JSON requests. -.. class:: JSONRequest(url, [... data]) +.. class:: JSONRequest(url, [... data, dumps_kwargs]) The :class:`JSONRequest` class adds two new argument to the constructor. The remaining arguments are the same as for the :class:`Request` class and are From f6dfc5f3dd56b7c823e3f53f7f9f63515ca7c3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:23:23 -0300 Subject: [PATCH 0217/1853] Fix boto import error under Jessie testing environment --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4218d13bf..252c783d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,11 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [ "$TOXENV" = "jessie" ]; then + # Not used directly but allows boto GCE plugins to load. + # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + pip install google-compute-engine + fi - pip install -U tox twine wheel codecov script: tox From 8ed6beb7f9199e8924cd03bd34a46194c3d82e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:39:29 -0300 Subject: [PATCH 0218/1853] Needs to be installed within tox env --- .travis.yml | 5 ----- tox.ini | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 252c783d7..4218d13bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,11 +43,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [ "$TOXENV" = "jessie" ]; then - # Not used directly but allows boto GCE plugins to load. - # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 - pip install google-compute-engine - fi - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e5543fe2a..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -51,6 +51,9 @@ deps = cssselect==0.9.1 zope.interface==4.1.1 -rtests/requirements-py2.txt +# Not used directly but allows boto GCE plugins to load. +# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + google-compute-engine==2.8.12 [testenv:trunk] basepython = python2.7 From f85c915872cf70bb87a05cecd6ef5a6534d2c4ed Mon Sep 17 00:00:00 2001 From: Joaquin Garmendia Cabrera Date: Sun, 23 Dec 2018 00:26:58 -0500 Subject: [PATCH 0219/1853] Update item-pipeline example --- docs/topics/item-pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 38265b474..1c2c51e05 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -87,8 +87,8 @@ contain a price:: vat_factor = 1.15 def process_item(self, item, spider): - if item['price']: - if item['price_excludes_vat']: + if 'price' in item and item['price']: + if 'price_excludes_vat' in item and item['price_excludes_vat']: item['price'] = item['price'] * self.vat_factor return item else: From 7c26701012c8e41a3e2c2644e05ce852d7472bc3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 01:33:58 +0500 Subject: [PATCH 0220/1853] DOC warn about telnet console being insecure --- docs/topics/telnetconsole.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 4db9cafb2..bf2ffa443 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -16,6 +16,17 @@ The telnet console is a :ref:`built-in Scrapy extension disable it if you want. For more information about the extension itself see :ref:`topics-extensions-ref-telnetconsole`. +.. warning:: + It is not secure to use telnet console via public networks, as telnet + doesn't provide any transport-layer security. Having username/password + authentication doesn't change that. + + Intended usage is connecting to a running Scrapy spider locally + (spider process and telnet client are on the same machine) + or over a secure connection (VPN, SSH tunnel). + Please avoid using telnet console over insecure connections, + or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option. + .. highlight:: none How to access the telnet console @@ -39,7 +50,12 @@ autogenerated Password can be seen on scrapy logs like the example bellow:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 Default Username and Password can be overriden by the settings -:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD` +:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. + +.. warning:: + Username and password provide only a limited protection, as telnet + is not using secure transport - by default traffic is not encrypted + even if username and password are set. You need the telnet program which comes installed by default in Windows, and most Linux distros. From cdd04dfb1d9a2e6fd8c188dccae26bbdd3454ebd Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 13:13:49 +0500 Subject: [PATCH 0221/1853] declare Python 3.7 support in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8c47f67ce..bd666e93c 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 71e47629b1cb65a61d8e4809177817c1a833f73c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 16:35:05 +0500 Subject: [PATCH 0222/1853] DOC fix docs for AWS_... settings. A follow-up to GH-2609. --- docs/topics/settings.rst | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 47b6cf13d..0ac26a9bd 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -178,35 +178,48 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. -.. setting:: BOT_NAME +.. setting:: AWS_ENDPOINT_URL AWS_ENDPOINT_URL ---------------- Default: ``None`` -Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality. +Endpoint URL used for S3-like storage, for example Minio or s3.scality. +Only supported with ``botocore`` library. -.. setting:: AWS_ENDPOINT_URL +.. setting:: AWS_USE_SSL AWS_USE_SSL ----------- Default: ``None`` -Use this option if you want to disable SSL connection for communication with S3 or S3-like storage. -By default SSL will be used. +Use this option if you want to disable SSL connection for communication with +S3 or S3-like storage. By default SSL will be used. +Only supported with ``botocore`` library. -.. setting:: AWS_USE_SSL +.. setting:: AWS_VERIFY AWS_VERIFY ---------- Default: ``None`` -Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur. +Verify SSL connection between Scrapy and S3 or S3-like storage. By default +SSL verification will occur. Only supported with ``botocore`` library. -.. setting:: AWS_VERIFY +.. setting:: AWS_REGION_NAME + +AWS_REGION_NAME +--------------- + +Default: ``None`` + +The name of the region associated with the AWS client. +Only supported with ``botocore`` library. + +.. setting:: BOT_NAME BOT_NAME -------- From a5e1b7bb4724bafa26b476a87a9f12b4d6479661 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 29 Nov 2018 18:19:14 -0300 Subject: [PATCH 0223/1853] add sitemap_filter attribute to SitemapSpider class it makes it possible to filter sitemap urls by any available attribute for example, you can filter urls with lastmod greater than a given datetime it can be helpful when the url loc itself does not aggregate that information --- docs/topics/spiders.rst | 26 ++++++++++++++++++++++++++ scrapy/spiders/sitemap.py | 10 ++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a08dc30f2..b0b9e0483 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,6 +680,32 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. attribute:: sitemap_filter + + Specifies a function to filter sitemap entries and their attributes. + + For example:: + + + http://example.com/ + 2005-01-01 + + + We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + + def sitemap_filter(urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield url + + This would retrieve only ``urls`` modified on 2005 and the following + years. + + If you omit this attribute, all urls found in sitemaps will be + processed, observing other attributes and their settings. + SitemapSpider examples ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0ee8ba5e7..907aba243 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,6 +17,7 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False + sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -43,12 +44,17 @@ class SitemapSpider(Spider): return s = Sitemap(body) + if callable(self.sitemap_filter): + it = self.sitemap_filter(s) + else: + it = s + if s.type == 'sitemapindex': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) From 672385a371453c84faa2f31425e3701b25260629 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 29 Nov 2018 18:33:20 -0300 Subject: [PATCH 0224/1853] using a method definition instead of a None attribute --- docs/topics/spiders.rst | 4 ++-- scrapy/spiders/sitemap.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0b9e0483..127c8d03e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. attribute:: sitemap_filter + .. method:: sitemap_filter(urls) Specifies a function to filter sitemap entries and their attributes. @@ -703,7 +703,7 @@ SitemapSpider This would retrieve only ``urls`` modified on 2005 and the following years. - If you omit this attribute, all urls found in sitemaps will be + If you omit this method, all urls found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 907aba243..c86e986db 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,7 +17,6 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False - sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -32,6 +31,14 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) + def sitemap_filter(self, urls): + """This method can be used to filter sitemap entries by their + attributes, for example, you can filter locs with lastmod greater + than a given date (see docs). + """ + for url in urls: + yield url + def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): for url in sitemap_urls_from_robots(response.text, base_url=response.url): @@ -44,10 +51,7 @@ class SitemapSpider(Spider): return s = Sitemap(body) - if callable(self.sitemap_filter): - it = self.sitemap_filter(s) - else: - it = s + it = self.sitemap_filter(s) if s.type == 'sitemapindex': for loc in iterloc(it, self.sitemap_alternate_links): From d7d5917ff12ecb8db7cd04592f7cc18b0ab1a996 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 30 Nov 2018 11:20:12 -0300 Subject: [PATCH 0225/1853] add tests for the sitemap_filter method in the SitemapSpider class --- tests/test_spider.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index f26da2334..871852ab2 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/italiano/']) + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year > 2008: + yield url + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/portuguese/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + class DeprecationTest(unittest.TestCase): From 657f0663b3cb97ca1c1a498c066de444bd30fa82 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 20 Dec 2018 13:35:52 -0300 Subject: [PATCH 0226/1853] rename param from urls to entries --- docs/topics/spiders.rst | 16 ++++++++-------- scrapy/spiders/sitemap.py | 6 +++--- tests/test_spider.py | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 127c8d03e..918f1cc36 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. method:: sitemap_filter(urls) + .. method:: sitemap_filter(entries) Specifies a function to filter sitemap entries and their attributes. @@ -691,19 +691,19 @@ SitemapSpider 2005-01-01 - We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - def sitemap_filter(urls): + def sitemap_filter(entries): from datetime import datetime - for url in urls: - date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year >= 2005: - yield url + yield entry - This would retrieve only ``urls`` modified on 2005 and the following + This would retrieve only ``entries`` modified on 2005 and the following years. - If you omit this method, all urls found in sitemaps will be + If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c86e986db..534c45c70 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -31,13 +31,13 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, urls): + def sitemap_filter(self, entries): """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ - for url in urls: - yield url + for entry in entries: + yield entry def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): diff --git a/tests/test_spider.py b/tests/test_spider.py index 871852ab2..d5d10c9ea 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml """ class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, urls): + def sitemap_filter(self, entries): from datetime import datetime - for url in urls: - date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year > 2008: - yield url + yield entry r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") From 5e7ecf9dc1954060fd0445dce5fb54e020dd3e59 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 21 Dec 2018 17:31:52 -0300 Subject: [PATCH 0227/1853] add tests for sitemapindex --- tests/test_spider.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index d5d10c9ea..8b56cfec1 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,37 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d') + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap1.xml', + 'http://www.example.com/sitemap2.xml']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap2.xml']) + class DeprecationTest(unittest.TestCase): From 10f46bca54b2879da02641159e53453fe0cc97dc Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 11:20:18 -0300 Subject: [PATCH 0228/1853] documenting sitemap entries as suggested by @kmike --- docs/topics/spiders.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 918f1cc36..9d4ed6ca6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -703,6 +703,16 @@ SitemapSpider This would retrieve only ``entries`` modified on 2005 and the following years. + Entries are dict objects extracted from the sitemap document. + Usually, the key is the tag name and the value is the text inside it. + + It's important to notice that: + + - as the loc attribute is required, entries without this tag are discarded + - alternate links are stored in a list with the key ``alternate`` + (see ``sitemap_alternate_links``) + - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar`` + If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. From fe283bcd058734f88977a2033dfa36664e7ee619 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 12:32:22 -0300 Subject: [PATCH 0229/1853] add test case for sitemap filter with alternate links --- tests/test_spider.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index 8b56cfec1..fefdaa403 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,41 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get('alternate', tuple()) + for link in alternate_links: + if '/deutsch/' in link: + entry['loc'] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/article_1/', + 'http://www.example.com/english/article_2/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/deutsch/article_1/']) + def test_sitemapindex_filter(self): sitemap = b""" From e1597f7c420ead9a563677aab61f18f9b89640a9 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 15:05:21 -0300 Subject: [PATCH 0230/1853] improve readability --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 9d4ed6ca6..c47a2fca0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -711,7 +711,7 @@ SitemapSpider - as the loc attribute is required, entries without this tag are discarded - alternate links are stored in a list with the key ``alternate`` (see ``sitemap_alternate_links``) - - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar`` + - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. From b68308779a6d2ce7deda3675d0bcdf671a4fb935 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 17:37:59 -0300 Subject: [PATCH 0231/1853] improving docs --- docs/topics/spiders.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c47a2fca0..4f7135309 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -682,7 +682,8 @@ SitemapSpider .. method:: sitemap_filter(entries) - Specifies a function to filter sitemap entries and their attributes. + This is a filter funtion that could be overridden to select sitemap entries + based on their attributes. For example:: @@ -693,12 +694,17 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - def sitemap_filter(entries): - from datetime import datetime - for entry in entries: - date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') - if date_time.year >= 2005: - yield entry + class FilteredSitemapSpider(scrapy.SitemapSpider): + name = 'filtered_sitemap_spider' + allowed_domains = ['example.com'] + sitemap_urls = ['http://example.com/sitemap.xml'] + + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield entry This would retrieve only ``entries`` modified on 2005 and the following years. From bfbcf52e9df77af7a7c9a8a7a711e06612be4763 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 18:12:31 -0300 Subject: [PATCH 0232/1853] fix SitemapSpider import --- docs/topics/spiders.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4f7135309..39410d66e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -694,7 +694,9 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - class FilteredSitemapSpider(scrapy.SitemapSpider): + from scrapy.spiders.sitemap import SitemapSpider + + class FilteredSitemapSpider(SitemapSpider): name = 'filtered_sitemap_spider' allowed_domains = ['example.com'] sitemap_urls = ['http://example.com/sitemap.xml'] From 5a824c906c501a204624ea7b4fb99904807c8b81 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 18:34:41 -0300 Subject: [PATCH 0233/1853] using shorter import version and moving datetime import to the beginning of the code snippet --- docs/topics/spiders.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 39410d66e..742a88659 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -694,7 +694,8 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - from scrapy.spiders.sitemap import SitemapSpider + from datetime import datetime + from scrapy.spiders import SitemapSpider class FilteredSitemapSpider(SitemapSpider): name = 'filtered_sitemap_spider' @@ -702,7 +703,6 @@ SitemapSpider sitemap_urls = ['http://example.com/sitemap.xml'] def sitemap_filter(self, entries): - from datetime import datetime for entry in entries: date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year >= 2005: From e1f8b55ba0a132ed28c71661e2df3c5bc27feb75 Mon Sep 17 00:00:00 2001 From: Joaquin Garmendia Cabrera Date: Fri, 28 Dec 2018 16:53:12 -0500 Subject: [PATCH 0234/1853] Improve syntax for readability --- docs/topics/item-pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 1c2c51e05..fae18200a 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -87,8 +87,8 @@ contain a price:: vat_factor = 1.15 def process_item(self, item, spider): - if 'price' in item and item['price']: - if 'price_excludes_vat' in item and item['price_excludes_vat']: + if item.get('price'): + if item.get('price_excludes_vat'): item['price'] = item['price'] * self.vat_factor return item else: From 6c78b3d5ef94791b11c2ce3dfd5cebd757a68b2a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 3 Jan 2019 13:15:58 -0300 Subject: [PATCH 0235/1853] Deques can't be sliced, use itertools.islice instead --- scrapy/core/spidermw.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 58bd7c2c8..e07f76bdf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from itertools import chain +from itertools import chain, islice import six from twisted.python.failure import Failure @@ -58,7 +58,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - method_list = self.methods['process_spider_exception'][start_index:] + method_list = islice(self.methods['process_spider_exception'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue @@ -89,7 +89,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recovered.extend(exception_result) - method_list = self.methods['process_spider_output'][start_index:] + method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue From 50a0d87d1e472fcc514f3dc2b028b653b7826a9c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 3 Jan 2019 17:20:08 -0300 Subject: [PATCH 0236/1853] Passing keyword arguments to callbacks --- scrapy/core/scraper.py | 2 +- scrapy/http/request/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ee1e95a0c..7981ce231 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -143,7 +143,7 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - dfd.addCallbacks(request.callback or spider.parse, request.errback) + dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index cd4360483..7d5cc9dae 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -18,7 +18,7 @@ class Request(object_ref): def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None): + dont_filter=False, errback=None, flags=None, kwargs=None): self._encoding = encoding # this one has to be set first self.method = str(method).upper() @@ -41,6 +41,7 @@ class Request(object_ref): self._meta = dict(meta) if meta else None self.flags = [] if flags is None else list(flags) + self.kwargs = dict(kwargs) if kwargs else None @property def meta(self): From a2b509a42266a2ab3389de64b608e616f88f77e5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 3 Jan 2019 17:38:06 -0300 Subject: [PATCH 0237/1853] Pass callback kwargs with response.follow --- scrapy/http/response/__init__.py | 5 +++-- scrapy/http/response/text.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 1974259b5..99b04a26e 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -106,7 +106,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None): + dont_filter=False, errback=None, kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -132,4 +132,5 @@ class Response(object_ref): encoding=encoding, priority=priority, dont_filter=dont_filter, - errback=errback) + errback=errback, + kwargs=kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 74a042f2c..2039621b3 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -123,7 +123,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None): + dont_filter=False, errback=None, kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -154,7 +154,8 @@ class TextResponse(Response): encoding=encoding, priority=priority, dont_filter=dont_filter, - errback=errback + errback=errback, + kwargs=kwargs, ) From 69a1ee79aa43bb1444e5b2a800a2e1702db6866e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 3 Jan 2019 17:38:29 -0300 Subject: [PATCH 0238/1853] Copy request.kwargs --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7d5cc9dae..9a155f415 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -93,7 +93,7 @@ class Request(object_ref): given new values. """ for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback']: + 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From a67f1ce512ffa111a400e2718ebbf3e7bd32d0ae Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 3 Jan 2019 17:49:41 -0300 Subject: [PATCH 0239/1853] Serialize Request kwargs --- scrapy/utils/reqser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 959dddbd5..d537057b1 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -32,7 +32,8 @@ def request_to_dict(request, spider=None): '_encoding': request._encoding, 'priority': request.priority, 'dont_filter': request.dont_filter, - 'flags': request.flags + 'flags': request.flags, + 'kwargs': request.kwargs, } if type(request) is not Request: d['_class'] = request.__module__ + '.' + request.__class__.__name__ @@ -64,7 +65,9 @@ def request_from_dict(d, spider=None): encoding=d['_encoding'], priority=d['priority'], dont_filter=d['dont_filter'], - flags=d.get('flags')) + flags=d.get('flags'), + kwargs=d.get('kwargs'), + ) def _find_method(obj, func): From 770a501fb32b7582acaa5900ac2f41ea46a321cd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 9 Jan 2019 10:40:03 -0300 Subject: [PATCH 0240/1853] Test request kwargs (copy, serialization) --- scrapy/http/request/__init__.py | 8 +++++++- tests/test_http_request.py | 5 +++++ tests/test_utils_reqser.py | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9a155f415..c016eb727 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -40,8 +40,14 @@ class Request(object_ref): self.dont_filter = dont_filter self._meta = dict(meta) if meta else None + self._kwargs = dict(kwargs) if kwargs else None self.flags = [] if flags is None else list(flags) - self.kwargs = dict(kwargs) if kwargs else None + + @property + def kwargs(self): + if self._kwargs is None: + self._kwargs = {} + return self._kwargs @property def meta(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..610893d8a 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -177,6 +177,7 @@ class RequestTest(unittest.TestCase): r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], callback=somecallback, errback=somecallback) r1.meta['foo'] = 'bar' + r1.kwargs['key'] = 'value' r2 = r1.copy() # make sure copy does not propagate callbacks @@ -189,6 +190,10 @@ class RequestTest(unittest.TestCase): assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" self.assertEqual(r1.flags, r2.flags) + # make sure kwargs dict is shallow copied + assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical" + self.assertEqual(r1.kwargs, r2.kwargs) + # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" self.assertEqual(r1.meta, r2.meta) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index dcc070b8f..76de20f22 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -26,6 +26,7 @@ class RequestSerializationTest(unittest.TestCase): encoding='latin-1', priority=20, meta={'a': 'b'}, + kwargs={'k': 'v'}, flags=['testFlag']) self._assert_serializes_ok(r, spider=self.spider) @@ -52,6 +53,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) + self.assertEqual(r1.kwargs, r2.kwargs) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) From 57e7c769779b9d37058d1d8839215a9c269b8c5b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 9 Jan 2019 10:40:44 -0300 Subject: [PATCH 0241/1853] Test callback kwargs --- tests/spiders.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_crawl.py | 8 +++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 7816bf7c7..5a1471072 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -28,6 +28,40 @@ class MetaSpider(MockServerSpider): self.meta['close_reason'] = reason +class KeywordArgumentsSpider(MockServerSpider): + + name = 'kwargs' + checks = set() + + def start_requests(self): + data = {'key': 'value', 'number': 123} + yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data) + yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data) + yield Request(self.mockserver.url('/general_without'), self.parse_general) + yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) + + def parse_first(self, response, key, number): + self.checks.add(key == 'value') + self.checks.add(number == 123) + yield response.follow( + self.mockserver.url('/two'), + self.parse_second, + kwargs={'new_key': 'new_value'}) + + def parse_second(self, response, new_key): + self.checks.add(new_key == 'new_value') + + def parse_general(self, response, **kwargs): + if response.url.endswith('/general_with'): + self.checks.add(kwargs['key'] == 'value') + self.checks.add(kwargs['number'] == 123) + elif response.url.endswith('/general_without'): + self.checks.add(kwargs == {}) + + def parse_no_kwargs(self, response): + pass + + class FollowAllSpider(MetaSpider): name = 'follow' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3fc13eeb7..3879a017c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -8,7 +8,7 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.crawler import CrawlerRunner from scrapy.utils.python import to_unicode -from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ +from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, KeywordArgumentsSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer @@ -23,6 +23,12 @@ class CrawlTestCase(TestCase): def tearDown(self): self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks + def test_callback_kwargs(self): + crawler = self.runner.create_crawler(KeywordArgumentsSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(crawler.spider.checks, set([True])) + @defer.inlineCallbacks def test_follow_all(self): crawler = self.runner.create_crawler(FollowAllSpider) From ac111088c6470d55e4d698247a76a4938f4a6dc6 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 20:12:29 +0530 Subject: [PATCH 0242/1853] duplicate keys handled --- scrapy/http/request/form.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c2413b431..660f4fc96 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode +from six.moves.urllib.parse import urljoin, urlencode, urlsplit import lxml.html from parsel.selector import create_root_node @@ -33,7 +33,30 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(querystr) else: - self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) + if urlsplit(self.url).query: + queries = (urlsplit(self.url).query + '&' + querystr).split('&') + else: + queries = querystr.split('&') + query_dict = {} + for i in range(len(queries)): + query_list = queries[i].split('=') + query_dict[query_list[0]] = query_list[1] + querystr = '' + query_key = list(query_dict.keys()) + for i in range(len(query_dict)): + querystr += (query_key[i] + '=' + query_dict[query_key[i]]) + if i!=len(query_dict)-1: + querystr += '&' + if urlsplit(self.url).fragment: + if urlsplit(self.url).query: + self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment) + else: + self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment) + else: + if urlsplit(self.url).query: + self._set_url(self.url[:self.url.index('?')+1]+querystr) + else: + self._set_url(self.url+'?'+querystr) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From b5e454809e240839e76f050b4587abcd80c33765 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 20:12:31 +0530 Subject: [PATCH 0243/1853] Included test --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..5a7cf3918 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -273,6 +273,19 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') + + def test_formdata_overrides_querystring_duplicates(self): + data1 = {'a' : '1', 'b' : '2'} + data2 = [('a','one'), ('a','two'), ('c','three')] + + fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) + self.assertEqual(fs1[b'a'], [b'1']) + self.assertEqual(fs1[b'b'], [b'2']) + + fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2)) + self.assertEqual(fs2[b'a'], [b'two']) + self.assertEqual(fs2[b'b'], [b'2']) + self.assertEqual(fs2[b'c'], [b'three']) def test_default_encoding_bytes(self): # using default encoding (utf-8) From 1bea5d307628a11d4010cdabb87b2a29a6c3c772 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 22:35:16 +0530 Subject: [PATCH 0244/1853] Fixed error --- scrapy/http/request/form.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 660f4fc96..f08d1f626 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,29 +34,32 @@ class FormRequest(Request): self._set_body(querystr) else: if urlsplit(self.url).query: - queries = (urlsplit(self.url).query + '&' + querystr).split('&') + queries = (urlsplit(self.url).query).split('&') else: queries = querystr.split('&') query_dict = {} + duplicate_key_passed=[] + for i in range(len(querystr.split('&'))): + query_list = querystr.split('&')[i].split('=') + duplicate_key_passed.append(query_list[0]) for i in range(len(queries)): query_list = queries[i].split('=') - query_dict[query_list[0]] = query_list[1] - querystr = '' + if duplicate_key_passed.count(query_list[0])<=1: + query_dict[query_list[0]] = query_list[1] + query_str = '' query_key = list(query_dict.keys()) for i in range(len(query_dict)): - querystr += (query_key[i] + '=' + query_dict[query_key[i]]) - if i!=len(query_dict)-1: - querystr += '&' + query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&') if urlsplit(self.url).fragment: if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment) + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment) else: - self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment) + self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment) else: if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1]+querystr) + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr) else: - self._set_url(self.url+'?'+querystr) + self._set_url(self.url + '?' + querystr) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 9a4bbd6d029f61dc60594fb8bca631c3a494f855 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:05:58 +0530 Subject: [PATCH 0245/1853] Update form.py --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f08d1f626..1b487cbd0 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -44,7 +44,7 @@ class FormRequest(Request): duplicate_key_passed.append(query_list[0]) for i in range(len(queries)): query_list = queries[i].split('=') - if duplicate_key_passed.count(query_list[0])<=1: + if duplicate_key_passed.count(query_list[0])==0: query_dict[query_list[0]] = query_list[1] query_str = '' query_key = list(query_dict.keys()) From 4abcdcb306d8782e6b792ad36e40391a0bcbd0f3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:22:53 +0530 Subject: [PATCH 0246/1853] Update test_http_request.py --- tests/test_http_request.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5a7cf3918..ec247a02d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -276,17 +276,10 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): data1 = {'a' : '1', 'b' : '2'} - data2 = [('a','one'), ('a','two'), ('c','three')] - - fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) + fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) self.assertEqual(fs1[b'a'], [b'1']) self.assertEqual(fs1[b'b'], [b'2']) - fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2)) - self.assertEqual(fs2[b'a'], [b'two']) - self.assertEqual(fs2[b'b'], [b'2']) - self.assertEqual(fs2[b'c'], [b'three']) - def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From 023290dabc55c92ea2143a98f7753abaf21f74c9 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:50:31 +0530 Subject: [PATCH 0247/1853] Update test_http_request.py --- tests/test_http_request.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ec247a02d..1ef84f1d6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,10 +275,10 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - data1 = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) - self.assertEqual(fs1[b'a'], [b'1']) - self.assertEqual(fs1[b'b'], [b'2']) + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) def test_default_encoding_bytes(self): # using default encoding (utf-8) From 3f914f6d8c369a18e1f856c01b7d1ad2a63f6e49 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Mon, 14 Jan 2019 23:03:14 +0530 Subject: [PATCH 0248/1853] made jsonrequest dump into private method --- scrapy/http/request/json_request.py | 15 +++++++++------ tests/test_http_request.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 7499610b9..1e2c6b0c6 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,6 +5,7 @@ This module implements the JSONRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +import copy import json import warnings @@ -13,7 +14,10 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): - dumps_kwargs = kwargs.pop('dumps_kwargs', {}) + dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + dumps_kwargs['sort_keys'] = True + self._dumps_kwargs = dumps_kwargs + body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None @@ -22,7 +26,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = self.dump(data, **dumps_kwargs) + kwargs['body'] = self._dumps(data) if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -30,7 +34,6 @@ class JSONRequest(Request): super(JSONRequest, self).__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') - self._dumps_kwargs = dumps_kwargs def replace(self, *args, **kwargs): body_passed = kwargs.get('body', None) is not None @@ -41,10 +44,10 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = self.dump(data, **self._dumps_kwargs) + kwargs['body'] = self._dumps(data) return super(JSONRequest, self).replace(*args, **kwargs) - def dump(self, data, **kwargs): + def _dumps(self, data): """Convert to JSON """ - return json.dumps(data, sort_keys=True, **kwargs) + return json.dumps(data, **self._dumps_kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dcfa25da..49f148016 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1292,7 +1292,7 @@ class JSONRequestTest(RequestTest): self.assertEqual(kwargs['sort_keys'], True) def test_replace_dumps_kwargs(self): - """ Test that dumps_kwargs are provided json.dumps when replace is called """ + """ Test that dumps_kwargs are provided to json.dumps when replace is called """ data1 = { 'name1': 'value1', } From bddfeaba4c17040b2986403f8b2ba25d4252e1b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 15 Jan 2019 15:35:46 -0300 Subject: [PATCH 0249/1853] Add Request.kwargs docs --- docs/topics/request-response.rst | 40 +++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e29914dbf..d12766676 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags]) +.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -126,6 +126,9 @@ Request objects :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list + :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback. + :type kwargs: dict + .. attribute:: Request.url A string containing the URL of this request. Keep in mind that this @@ -165,6 +168,17 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. + .. attribute:: Request.kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a :class:`Response` + object as argument. + + This dict is `shallow copied`_ when the request is cloned using the + ``copy()`` or ``replace()`` methods, and can also be accessed, in your + spider, from the ``response.kwargs`` attribute. + .. _shallow copied: https://docs.python.org/2/library/copy.html .. method:: Request.copy() @@ -200,11 +214,9 @@ Example:: self.logger.info("Visited %s", response.url) In some cases you may be interested in passing arguments to those callback -functions so you can receive the arguments later, in the second callback. You -can use the :attr:`Request.meta` attribute for that. - -Here's an example of how to pass an item using this mechanism, to populate -different fields from different pages:: +functions so you can receive the arguments later, in the second callback. +The following two examples show how to achieve this by using the +:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively:: def parse_page1(self, response): item = MyItem() @@ -219,6 +231,22 @@ different fields from different pages:: item['other_url'] = response.url yield item +:: + + def parse_page1(self, response): + item = MyItem() + item['main_url'] = response.url + request = scrapy.Request("http://www.example.com/some_page.html", + callback=self.parse_page2) + request.kwargs['item'] = item + request.kwargs['foo'] = 'bar' + yield request + + def parse_page2(self, response, item, foo): + item['other_url'] = response.url + item['foo'] = foo + yield item + .. _topics-request-response-ref-errbacks: From 9f1f4df9666f38f81adda5a41f3cac646a4f18dc Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 22:59:41 +0530 Subject: [PATCH 0250/1853] Update test_http_request.py --- tests/test_http_request.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1ef84f1d6..6c4cb10a8 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,8 +275,22 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): + #Without fragment data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data)) + fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #With fragment + data = (('a', '1'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #Witout query + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) From 3e67fa8fc1eaa2189cec08ce4000b3efc254ee1b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:01:38 +0530 Subject: [PATCH 0251/1853] Improved for better user readability --- scrapy/http/request/form.py | 45 ++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1b487cbd0..fa5110382 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -33,33 +33,26 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(querystr) else: - if urlsplit(self.url).query: - queries = (urlsplit(self.url).query).split('&') + url_split = urlsplit(self.url) + formdata_key_list = [] + for k in querystr.split('&'): + formdata_key_list.append(k.split('=')[0]) + + if url_split.query: + queries = (url_split.query).split('&') + query_dict = {} + + for x in queries: + query = x.split('=') + if formdata_key_list.count(query[0])==0: + query_dict[query[0]] = query[1] + query_str = '' + for k, v in query_dict: + query_str += (k + '=' + v + '&') + + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) else: - queries = querystr.split('&') - query_dict = {} - duplicate_key_passed=[] - for i in range(len(querystr.split('&'))): - query_list = querystr.split('&')[i].split('=') - duplicate_key_passed.append(query_list[0]) - for i in range(len(queries)): - query_list = queries[i].split('=') - if duplicate_key_passed.count(query_list[0])==0: - query_dict[query_list[0]] = query_list[1] - query_str = '' - query_key = list(query_dict.keys()) - for i in range(len(query_dict)): - query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&') - if urlsplit(self.url).fragment: - if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment) - else: - self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment) - else: - if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr) - else: - self._set_url(self.url + '?' + querystr) + self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 6f86c93f366509ba05faf3b2cdc12f5e3ef6693a Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:54:35 +0530 Subject: [PATCH 0252/1853] Increased test cases --- tests/test_http_request.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6c4cb10a8..571df0176 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,20 +275,27 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - #Without fragment - data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #With fragment + #Both fragment and query in url data = (('a', '1'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #Witout query + #only query in url + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #only fragment in url + data = (('a', '1'), ('b', '2')) + url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #None of both in url data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) From 6be73f06c33a27cb3291a1e46d80574b4d23b10b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Thu, 17 Jan 2019 23:50:58 +0530 Subject: [PATCH 0253/1853] Updated tests --- tests/test_http_request.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 571df0176..a17acf93b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -277,23 +277,11 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): #Both fragment and query in url data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] - fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #only query in url - data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #only fragment in url - data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0] + url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) + self.assertEqual(fs[b'c'], [b'3']) #None of both in url data = {'a' : '1', 'b' : '2'} From a9f68acb6dd7f5aca2bee95353278e5a40c32b8b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Thu, 17 Jan 2019 23:51:09 +0530 Subject: [PATCH 0254/1853] modified code --- scrapy/http/request/form.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index fa5110382..2305491e2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -41,15 +41,14 @@ class FormRequest(Request): if url_split.query: queries = (url_split.query).split('&') query_dict = {} - for x in queries: - query = x.split('=') - if formdata_key_list.count(query[0])==0: - query_dict[query[0]] = query[1] + k = x.split('=')[0] + v = x.split('=')[1] + if formdata_key_list.count(k)==0: + query_dict[k] = v query_str = '' - for k, v in query_dict: + for k, v in query_dict.items(): query_str += (k + '=' + v + '&') - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) else: self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) From bdf12f775062fda8aa8bf03f7b4faade4faac16d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 18 Jan 2019 11:38:59 -0200 Subject: [PATCH 0255/1853] Logging the request referer when DUPEFILTER_DEBUG is active --- scrapy/dupefilters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 9d8966b9c..0bcdd3495 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -3,8 +3,7 @@ import os import logging from scrapy.utils.job import job_dir -from scrapy.utils.request import request_fingerprint - +from scrapy.utils.request import referer_str, request_fingerprint class BaseDupeFilter(object): @@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter): def log(self, request, spider): if self.debug: - msg = "Filtered duplicate request: %(request)s" - self.logger.debug(msg, {'request': request}, extra={'spider': spider}) + msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" + args = {'request': request, 'referer': referer_str(request) } + self.logger.debug(msg, args, extra={'spider': spider}) elif self.logdupes: msg = ("Filtered duplicate request: %(request)s" " - no more duplicates will be shown" From 8eade7d8640e112faf8677f4666bbe3ab10c7234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 18 Jan 2019 11:39:35 -0200 Subject: [PATCH 0256/1853] Testing stats and log messages from RFPDupeFilter --- tests/test_dupefilters.py | 57 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index db69597a2..d7eb98c97 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,6 +2,7 @@ import hashlib import tempfile import unittest import shutil +from testfixtures import LogCapture from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request @@ -9,7 +10,7 @@ from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler - +from tests.spiders import SimpleSpider class FromCrawlerRFPDupeFilter(RFPDupeFilter): @@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase): assert case_insensitive_dupefilter.request_seen(r2) case_insensitive_dupefilter.close('finished') + + def test_log(self): + with LogCapture() as l: + settings = {'DUPEFILTER_DEBUG': False, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + spider = SimpleSpider.from_crawler(crawler) + + dupefilter = scheduler.df + dupefilter.open() + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html') + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' - no more duplicates will be shown' + ' (see DUPEFILTER_DEBUG to show all duplicates)'))) + + dupefilter.close('finished') + + def test_log_debug(self): + with LogCapture() as l: + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + spider = SimpleSpider.from_crawler(crawler) + + dupefilter = scheduler.df + dupefilter.open() + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html', + headers={'Referer': 'http://scrapytest.org/INDEX.html'} + ) + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' (referer: None)'))) + l.check_present(('scrapy.dupefilters', 'DEBUG', + ('Filtered duplicate request: ' + ' (referer: http://scrapytest.org/INDEX.html)'))) + + dupefilter.close('finished') From 7dee841b8b904a3961481f7b1999d819d2c17c5d Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sat, 19 Jan 2019 13:20:01 +0530 Subject: [PATCH 0257/1853] Update form.py --- scrapy/http/request/form.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2305491e2..af4ed1793 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl import lxml.html from parsel.selector import create_root_node @@ -37,21 +37,10 @@ class FormRequest(Request): formdata_key_list = [] for k in querystr.split('&'): formdata_key_list.append(k.split('=')[0]) - - if url_split.query: - queries = (url_split.query).split('&') - query_dict = {} - for x in queries: - k = x.split('=')[0] - v = x.split('=')[1] - if formdata_key_list.count(k)==0: - query_dict[k] = v - query_str = '' - for k, v in query_dict.items(): - query_str += (k + '=' + v + '&') - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) - else: - self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) + items = [] + items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + query_str = _urlencode(items, self.encoding) + self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 722a30ac2bc35bdc7abb52d10417ed1191a3aee0 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sat, 19 Jan 2019 13:20:05 +0530 Subject: [PATCH 0258/1853] Update test_http_request.py --- tests/test_http_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a17acf93b..955db9e76 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -276,10 +276,10 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): #Both fragment and query in url - data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] + data = (('a', 'one'), ('a', 'two'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) self.assertEqual(fs[b'c'], [b'3']) From 71743a6546e96b5d99bd3c068a7ec5b71dca1659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 19 Jan 2019 18:43:58 +0000 Subject: [PATCH 0259/1853] Add release notes for v1.5.2 --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 01016e2e6..adf679ded 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,6 +12,22 @@ Cleanups * Remove deprecated ``CrawlerSettings`` class. * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. +Scrapy 1.5.2 (2019-01-22) +------------------------- + +* *Security bugfix*: Telnet console extension can be easily exploited by rogue + websites POSTing content to http://localhost:6023, we haven't found a way to + exploit it from Scrapy, but it is very easy to trick a browser to do so and + elevates the risk for local development environment. + + *The fix is backwards incompatible*, it enables telnet user-password + authentication by default with a random generated password. If you can't + upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT` + out of its default value. + + See :ref:`telnet console ` documentation for more info + +* Backport CI build failure under GCE environemnt due to boto import error. Scrapy 1.5.1 (2018-07-12) ------------------------- From d9aa5391327dd34f8d840e7ce2bca1eb8583d932 Mon Sep 17 00:00:00 2001 From: kasun Herath Date: Fri, 25 Jan 2019 21:26:28 +0530 Subject: [PATCH 0260/1853] enabled sort keys only if not provided --- scrapy/http/request/json_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1e2c6b0c6..8f7a61a6d 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -15,7 +15,7 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) - dumps_kwargs['sort_keys'] = True + dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs body_passed = kwargs.get('body', None) is not None From b828b5f8c8650a30aef382af661b6c7b9ea57186 Mon Sep 17 00:00:00 2001 From: Harry Moreno Date: Sat, 26 Jan 2019 18:39:05 -0500 Subject: [PATCH 0261/1853] fix grammar --- docs/topics/jobs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 8e1574376..ea684b4cf 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -30,7 +30,7 @@ a *single* job. How to use it ============= -To start a spider with persistence supported enabled, run it like this:: +To start a spider with persistence support enabled, run it like this:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 From 8fca98616a90d8452ff2e488bfea93e5c89caf08 Mon Sep 17 00:00:00 2001 From: Harry Moreno Date: Sat, 26 Jan 2019 16:47:10 -0500 Subject: [PATCH 0262/1853] fix grammar --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index a1f518cbd..c60b55391 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -310,7 +310,7 @@ images. .. setting:: IMAGES_THUMBS -In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary +In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary where the keys are the thumbnail names and the values are their dimensions. For example:: From e3e804cfb0fc05ef3fc569ec6e0af247ce504d06 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Jan 2019 15:10:34 -0300 Subject: [PATCH 0263/1853] Styling nitpick :-) --- scrapy/core/downloader/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a8e6f93a3..7a6a4dfac 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -41,7 +41,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) - defer.returnValue((yield download_func(request=request,spider=spider))) + defer.returnValue((yield download_func(request=request, spider=spider))) @defer.inlineCallbacks def process_response(response): From 706910790b6ee755bafa828606e215e668af3eee Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 18:28:24 +0500 Subject: [PATCH 0264/1853] [wip] draft 1.6 release notes --- docs/news.rst | 153 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index adf679ded..99a339cea 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,11 +6,156 @@ Release notes Scrapy 1.6.0 (unreleased) ------------------------- +Highlights for this release: + +* better Windows compatibility; +* Python 3.7 compatibility; +* big documentation improvements, including a switch + from ``.extract() / .extract_first()`` API to ``.get() / .getall()`` API; +* Feed exports, FilePipeline and MediaPipeline improvements; +* ``scrapy.contracts`` fixes and new features; +* large clean-up of deprecated code +* TODO + +parsel 1.5 +~~~~~~~~~~ + +TODO +While this is not a change in Scrapy itself, a new version of ``parsel`` +is released; Scrapy now depends on ``parsel >= 1.5``. + +Feed export improvements +~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``from_crawler`` support is added to feed exporters and feed storages. This, + among other things, allow to access Scrapy settings from custom storages + and exporters (:issue:`1605`, :issue:`3348`). +* fixed issue with extra blank lines in .csv exports under Windows + (:issue:`3039`); +* better error message when an exporter is disabled (:issue:`3358`); + +FilePipeline and MediaPipeline improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`, + :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`. + For example, this allows to use alternative or self-hosted + AWS-compatible providers (:issue:`2609`). +* ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and + :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`). + +``scrapy.contracts`` improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Exceptions in contracts code are handled better (:issue:`3377`); +* ``dont_filter=True`` is used for contract requests, which allows to test + different callbacks with the same URL (:issue:`3381`); +* ``request_cls`` attribute in Contract subclasses allow to use different + Request classes in contracts, for example FormRequest (:issue:`3383`). +* Fixed errback handling in contracts, e.g. for cases where a contract + is executed for URL which returns non-200 response (:issue:`3371`). + +Documentation improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Docs are re-written to suggest .get/.getall API instead of + .extract/.extract_first. Also, :ref:`topics-selectors` docs are updated + and re-structured to match latest parsel docs; they now contain more topics, + such as :ref:`selecting-attributes` or :ref:`topics-selectors-css-extensions` + (:issue:`3390`). +* :ref:`topics-developer-tools` is a new tutorial which replaces + old Firefox and Firebug tutorials (:issue:`3400`). +* SCRAPY_PROJECT environment variable is documented (:issue:`3518`); +* troubleshooting section is added to install instructions (:issue:`3517`); +* improved links to beginner resources in the tutorial + (:issue:`3367`, :issue:`3468`); +* fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); +* remove unused `DEPTH_STATS` option from docs (:issue:`3245`); +* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`). + +Better Windows support +~~~~~~~~~~~~~~~~~~~~~~ + +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Scrapy used to produce unnecessary blank lines in .csv exports on Windows, + this is fixed (:issue:`3039`). + +Testing fixes +~~~~~~~~~~~~~ + +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`) +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +Compatibility shims for pre-1.0 Scrapy module names are removed +(:issue:`3318`): + +* ``scrapy.command`` +* ``scrapy.contrib`` (with all submodules) +* ``scrapy.contrib_exp`` (with all submodules) +* ``scrapy.dupefilter`` +* ``scrapy.linkextractor`` +* ``scrapy.project`` +* ``scrapy.spider`` +* ``scrapy.spidermanager`` +* ``scrapy.squeue`` +* ``scrapy.stats`` +* ``scrapy.statscol`` +* ``scrapy.utils.decorator`` + +See :ref:`module_relocations` for more information, or use suggestions +from Scrapy 1.5.x deprecation warnings to update your code. + +Other deprecation removals: + +* Deprecated scrapy.interfaces.ISpiderManager is removed; please use + scrapy.interfaces.ISpiderLoader. +* Deprecated ``CrawlerSettings`` class is removed (:issue:`3327`). +* Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes + are removed (:issue:`3327`, :issue:`3359`). + +Internal improvements +~~~~~~~~~~~~~~~~~~~~~ + +* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows + to access e.g. settings or a spider from a dupefilter. +* :signal:`item_error` is fired when an error happens in a pipeline + (:issue:`3256`); +* :signal:`request_reached_downloader` is fired when Downloader gets + a new Request; this signal can be useful e.g. for custom Schedulers + (:issue:`3393`). +* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" + optional arguments (:issue:`3231`). + +Usability improvements +~~~~~~~~~~~~~~~~~~~~~~ + +* more stats for RobotsTxtMiddleware (:issue:`3100`) +* INFO log level is used to show telnet host/port (:issue:`3115`) +* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) +* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) +* non-zero exit code is returned from Scrapy commands when error happens + on spider inititalization (:issue:`3226`) +* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + "flv" is added to common video extensions (:issue:`3165`) + +Bug fixes +~~~~~~~~~ +* proper handling of pickling errors in Python 3 when serializing objects + for disk queues (:issue:`3082`) +* flags are now preserved when copying Requests (:issue:`3342`); +* FormRequest.from_response clickdata shouldn't ignore elements with + ``input[type=image]`` (:issue:`3153`). +* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) + Cleanups ~~~~~~~~ - -* Remove deprecated ``CrawlerSettings`` class. -* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. +* additional files are included to sdist (:issue:`3495`); +* code style fixes (:issue:`3405`, :issue:`3304`) Scrapy 1.5.2 (2019-01-22) ------------------------- @@ -1080,6 +1225,8 @@ until it reaches a stable status. See more examples for scripts running Scrapy: :ref:`topics-practices` +.. _module_relocations: + Module Relocations ~~~~~~~~~~~~~~~~~~ From e479f5aa15809e7f75a7dbc20d0629f57be46b5d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 27 Dec 2018 00:48:10 +0500 Subject: [PATCH 0265/1853] DOC update changelog * changes from recently merged pull requests * more highlights * re-organized headers * Selector API changes --- docs/news.rst | 142 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 105 insertions(+), 37 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 99a339cea..bf469a350 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,41 +6,83 @@ Release notes Scrapy 1.6.0 (unreleased) ------------------------- -Highlights for this release: +Highlights: -* better Windows compatibility; +* better Windows support; * Python 3.7 compatibility; * big documentation improvements, including a switch - from ``.extract() / .extract_first()`` API to ``.get() / .getall()`` API; -* Feed exports, FilePipeline and MediaPipeline improvements; + from ``.extract()`` + ``.extract_first()`` API to ``.get()`` + ``.getall()`` + API; +* feed exports, FilePipeline and MediaPipeline improvements; +* better extensibility: :signal:`item_error` and + :signal:`request_reached_downloader` signals; ``from_crawler`` support + for feed exporters, feed storages and dupefilters. * ``scrapy.contracts`` fixes and new features; -* large clean-up of deprecated code -* TODO +* telnet console security improvements; +* clean-up of the deprecated code; +* various bug fixes, small new features and usability improvements across + the codebase. -parsel 1.5 -~~~~~~~~~~ +Selector API changes +~~~~~~~~~~~~~~~~~~~~ -TODO -While this is not a change in Scrapy itself, a new version of ``parsel`` -is released; Scrapy now depends on ``parsel >= 1.5``. +While these are not changes in Scrapy itself, but rather in the parsel_ +library which Scrapy uses for xpath/css selectors, these changes are +worth mentioning here. Scrapy now depends on parsel >= 1.5, and +Scrapy documentation is updated to follow recent ``parsel`` API conventions. -Feed export improvements -~~~~~~~~~~~~~~~~~~~~~~~~ +Most visible change is that ``.get()`` and ``.getall()`` selector +methods are now preferred over ``.extract()`` and ``.extract_first()``. +We feel that these new methods result in a more concise and readable code. +See :ref:`old-extraction-api` for more details. + +.. note:: + There are currently **no plans** to deprecate ``.extract()`` + and ``.extract_first()`` methods. + +Another useful new feature is the introduction of ``Selector.attrib`` and +``SelectorList.attrib`` properties, which make it easier to get +attributes of HTML elements. See :ref:`selecting-attributes`. + +CSS selectors are cached in parsel >= 1.5, which makes them faster +when the same CSS path is used many times. This is very common in +case of Scrapy spiders: callbacks are usually called several times, +on different pages. + +If you're using custom ``Selector`` or ``SelectorList`` subclasses, +a **backwards incompatible** change in parsel may affect your code. +See `parsel changelog`_ for a detailed description, as well as for the +full list of improvements. + +.. _parsel changelog: https://parsel.readthedocs.io/en/latest/history.html + +Telnet console +~~~~~~~~~~~~~~ + +**Backwards incompatible**: Scrapy's telnet console now requires username +and password. See :ref:`topics-telnetconsole` for more details. + +New extensibility features +~~~~~~~~~~~~~~~~~~~~~~~~~~ * ``from_crawler`` support is added to feed exporters and feed storages. This, - among other things, allow to access Scrapy settings from custom storages - and exporters (:issue:`1605`, :issue:`3348`). -* fixed issue with extra blank lines in .csv exports under Windows - (:issue:`3039`); -* better error message when an exporter is disabled (:issue:`3358`); + among other things, allows to access Scrapy settings from custom feed + storages and exporters (:issue:`1605`, :issue:`3348`). +* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows + to access e.g. settings or a spider from a dupefilter. +* :signal:`item_error` is fired when an error happens in a pipeline + (:issue:`3256`); +* :signal:`request_reached_downloader` is fired when Downloader gets + a new Request; this signal can be useful e.g. for custom Schedulers + (:issue:`3393`). -FilePipeline and MediaPipeline improvements +New FilePipeline and MediaPipeline features ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`. For example, this allows to use alternative or self-hosted - AWS-compatible providers (:issue:`2609`). + AWS-compatible providers (:issue:`2609`, :issue:`3548`). * ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`). @@ -55,6 +97,47 @@ FilePipeline and MediaPipeline improvements * Fixed errback handling in contracts, e.g. for cases where a contract is executed for URL which returns non-200 response (:issue:`3371`). +Usability and other improvements, cleanups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). +* Lazy loading of Downloader Handlers is now optional; this enables better + initialization error handling in custom Downloader Handlers (:issue:`3394`). +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) +* better error message when an exporter is disabled (:issue:`3358`); +* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" + optional arguments (:issue:`3231`). +* more stats for RobotsTxtMiddleware (:issue:`3100`) +* INFO log level is used to show telnet host/port (:issue:`3115`) +* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) +* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) +* non-zero exit code is returned from Scrapy commands when error happens + on spider inititalization (:issue:`3226`); +* link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + "flv" is added to common video extensions (:issue:`3165`) +* `scrapy shell --help` mentions syntax required for local files + (``./file.html``) - :issue:`3496`. +* additional files are included to sdist (:issue:`3495`); +* code style fixes (:issue:`3405`, :issue:`3304`); +* unneeded .strip() call is removed (:issue:`3519`); +* collections.deque is used to store MiddlewareManager methods instead + of a list (:issue:`3476`) + +Bug fixes +~~~~~~~~~ + +* fixed issue with extra blank lines in .csv exports under Windows + (:issue:`3039`); +* proper handling of pickling errors in Python 3 when serializing objects + for disk queues (:issue:`3082`) +* flags are now preserved when copying Requests (:issue:`3342`); +* FormRequest.from_response clickdata shouldn't ignore elements with + ``input[type=image]`` (:issue:`3153`). +* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) + Documentation improvements ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -73,21 +156,6 @@ Documentation improvements * remove unused `DEPTH_STATS` option from docs (:issue:`3245`); * other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`). -Better Windows support -~~~~~~~~~~~~~~~~~~~~~~ - -* All Scrapy tests now pass on Windows; Scrapy testing suite is executed - in a Windows environment on CI (:issue:`3315`). -* Scrapy used to produce unnecessary blank lines in .csv exports on Windows, - this is fixed (:issue:`3039`). - -Testing fixes -~~~~~~~~~~~~~ - -* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`) -* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, - :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) - Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -107,7 +175,7 @@ Compatibility shims for pre-1.0 Scrapy module names are removed * ``scrapy.statscol`` * ``scrapy.utils.decorator`` -See :ref:`module_relocations` for more information, or use suggestions +See :ref:`module-relocations` for more information, or use suggestions from Scrapy 1.5.x deprecation warnings to update your code. Other deprecation removals: @@ -1225,7 +1293,7 @@ until it reaches a stable status. See more examples for scripts running Scrapy: :ref:`topics-practices` -.. _module_relocations: +.. _module-relocations: Module Relocations ~~~~~~~~~~~~~~~~~~ From 638469f9efdcc104f7b1a1c1a9890694e0d41c68 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 28 Dec 2018 01:13:01 +0500 Subject: [PATCH 0266/1853] DOC extract_first/extract matches get/getall better Thanks @Gallaecio! --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index bf469a350..4a236f1b9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,7 +11,7 @@ Highlights: * better Windows support; * Python 3.7 compatibility; * big documentation improvements, including a switch - from ``.extract()`` + ``.extract_first()`` API to ``.get()`` + ``.getall()`` + from ``.extract_first()`` + ``.extract()`` API to ``.get()`` + ``.getall()`` API; * feed exports, FilePipeline and MediaPipeline improvements; * better extensibility: :signal:`item_error` and @@ -32,7 +32,7 @@ worth mentioning here. Scrapy now depends on parsel >= 1.5, and Scrapy documentation is updated to follow recent ``parsel`` API conventions. Most visible change is that ``.get()`` and ``.getall()`` selector -methods are now preferred over ``.extract()`` and ``.extract_first()``. +methods are now preferred over ``.extract_first()`` and ``.extract()``. We feel that these new methods result in a more concise and readable code. See :ref:`old-extraction-api` for more details. From 4cf4dd1d3e068e0df32f700c89d833cc7cd79b85 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 03:08:17 +0500 Subject: [PATCH 0267/1853] DOC add recent changes to changelog --- docs/news.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 4a236f1b9..1a08f93ec 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -75,6 +75,9 @@ New extensibility features * :signal:`request_reached_downloader` is fired when Downloader gets a new Request; this signal can be useful e.g. for custom Schedulers (:issue:`3393`). +* new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows + to select sitemap entries based on their attributes in SitemapSpider + subclasses (:issue:`3512`). New FilePipeline and MediaPipeline features ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -125,6 +128,7 @@ Usability and other improvements, cleanups * unneeded .strip() call is removed (:issue:`3519`); * collections.deque is used to store MiddlewareManager methods instead of a list (:issue:`3476`) +* Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) Bug fixes ~~~~~~~~~ @@ -154,7 +158,8 @@ Documentation improvements (:issue:`3367`, :issue:`3468`); * fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); * remove unused `DEPTH_STATS` option from docs (:issue:`3245`); -* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`). +* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`, + :issue:`3605`). Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 0fc9d705c271f5d87174143c09f95993e5a45797 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 03:28:19 +0500 Subject: [PATCH 0268/1853] DOC mention that telnet security improvements happened in 1.5.2 --- docs/news.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 1a08f93ec..a4f07efad 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,8 @@ Release notes ============= +.. _release-1.6.0: + Scrapy 1.6.0 (unreleased) ------------------------- @@ -18,11 +20,13 @@ Highlights: :signal:`request_reached_downloader` signals; ``from_crawler`` support for feed exporters, feed storages and dupefilters. * ``scrapy.contracts`` fixes and new features; -* telnet console security improvements; +* telnet console security improvements, first released as a + backport in :ref:`release-1.5.2`; * clean-up of the deprecated code; * various bug fixes, small new features and usability improvements across the codebase. + Selector API changes ~~~~~~~~~~~~~~~~~~~~ @@ -230,6 +234,8 @@ Cleanups * additional files are included to sdist (:issue:`3495`); * code style fixes (:issue:`3405`, :issue:`3304`) +.. _release-1.5.2: + Scrapy 1.5.2 (2019-01-22) ------------------------- @@ -247,6 +253,8 @@ Scrapy 1.5.2 (2019-01-22) * Backport CI build failure under GCE environemnt due to boto import error. +.. _release-1.5.1: + Scrapy 1.5.1 (2018-07-12) ------------------------- @@ -262,6 +270,9 @@ This is a maintenance release with important bug fixes, but no new features: :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`, :issue:`3294`). + +.. _release-1.5.0: + Scrapy 1.5.0 (2017-12-29) ------------------------- @@ -373,6 +384,7 @@ Docs - Document ``from_crawler`` methods for spider and downloader middlewares (:issue:`3019`) +.. _release-1.4.0: Scrapy 1.4.0 (2017-05-18) ------------------------- @@ -559,6 +571,8 @@ Documentation - Clarify ``allowed_domains`` example (:issue:`2670`) +.. _release-1.3.3: + Scrapy 1.3.3 (2017-03-10) ------------------------- @@ -571,6 +585,7 @@ Bug fixes A new setting is introduced to toggle between warning or exception if needed ; see :setting:`SPIDER_LOADER_WARN_ONLY` for details. +.. _release-1.3.2: Scrapy 1.3.2 (2017-02-13) ------------------------- @@ -582,6 +597,8 @@ Bug fixes - Use consistent selectors for author field in tutorial (:issue:`2551`). - Fix TLS compatibility in Twisted 17+ (:issue:`2558`) +.. _release-1.3.1: + Scrapy 1.3.1 (2017-02-08) ------------------------- @@ -630,6 +647,8 @@ Cleanups - Remove dead code supporting old Twisted versions (:issue:`2544`). +.. _release-1.3.0: + Scrapy 1.3.0 (2016-12-21) ------------------------- @@ -669,6 +688,7 @@ Dependencies & Cleanups - ``ChunkedTransferMiddleware`` is deprecated and removed from the default downloader middlewares. +.. _release-1.2.3: Scrapy 1.2.3 (2017-03-03) ------------------------- @@ -676,6 +696,8 @@ Scrapy 1.2.3 (2017-03-03) - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.2.2: + Scrapy 1.2.2 (2016-12-06) ------------------------- @@ -711,6 +733,8 @@ Other changes .. _conda-forge: https://anaconda.org/conda-forge/scrapy +.. _release-1.2.1: + Scrapy 1.2.1 (2016-10-21) ------------------------- @@ -735,6 +759,8 @@ Other changes - Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`). +.. _release-1.2.0: + Scrapy 1.2.0 (2016-10-03) ------------------------- @@ -803,12 +829,14 @@ Documentation - Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`). - Add StackOverflow as a support channel (:issue:`2257`). +.. _release-1.1.4: Scrapy 1.1.4 (2017-03-03) ------------------------- - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.1.3: Scrapy 1.1.3 (2016-09-22) ------------------------- @@ -826,6 +854,7 @@ Documentation rewritten to use http://toscrape.com websites (:issue:`2236`, :issue:`2249`, :issue:`2252`). +.. _release-1.1.2: Scrapy 1.1.2 (2016-08-18) ------------------------- @@ -840,6 +869,7 @@ Bug fixes - :setting:`IMAGES_EXPIRES` default value set back to 90 (the regression was introduced in 1.1.1) +.. _release-1.1.1: Scrapy 1.1.1 (2016-07-13) ------------------------- @@ -892,6 +922,7 @@ Tests - Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`) +.. _release-1.1.0: Scrapy 1.1.0 (2016-05-11) ------------------------- @@ -1081,12 +1112,14 @@ Bugfixes - HTTPS+CONNECT tunnels could get mixed up when using multiple proxies to same remote host (:issue:`1912`). +.. _release-1.0.7: Scrapy 1.0.7 (2017-03-03) ------------------------- - Packaging fix: disallow unsupported Twisted versions in setup.py +.. _release-1.0.6: Scrapy 1.0.6 (2016-05-04) ------------------------- @@ -1096,6 +1129,7 @@ Scrapy 1.0.6 (2016-05-04) - DOC: Support for Sphinx 1.4+ (:issue:`1893`) - DOC: Consistency in selectors examples (:issue:`1869`) +.. _release-1.0.5: Scrapy 1.0.5 (2016-02-04) ------------------------- @@ -1105,6 +1139,7 @@ Scrapy 1.0.5 (2016-02-04) - DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`) - DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`) +.. _release-1.0.4: Scrapy 1.0.4 (2015-12-30) ------------------------- @@ -1158,12 +1193,16 @@ Scrapy 1.0.4 (2015-12-30) - Small grammatical change (:commit:`8752294`) - Add openssl version to version command (:commit:`13c45ac`) +.. _release-1.0.3: + Scrapy 1.0.3 (2015-08-11) ------------------------- - add service_identity to scrapy install_requires (:commit:`cbc2501`) - Workaround for travis#296 (:commit:`66af9cd`) +.. _release-1.0.2: + Scrapy 1.0.2 (2015-08-06) ------------------------- @@ -1174,6 +1213,8 @@ Scrapy 1.0.2 (2015-08-06) - Fixed typos (:commit:`a9ae7b0`) - Fix doc reference. (:commit:`7c8a4fe`) +.. _release-1.0.1: + Scrapy 1.0.1 (2015-07-01) ------------------------- @@ -1184,6 +1225,8 @@ Scrapy 1.0.1 (2015-07-01) - DOC remove version suffix from ubuntu package (:commit:`5303c66`) - DOC Update release date for 1.0 (:commit:`c89fa29`) +.. _release-1.0.0: + Scrapy 1.0.0 (2015-06-19) ------------------------- From 2c8c8b2dd8683787826713ed1d0fbfb2ec1af04a Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 17:30:13 +0500 Subject: [PATCH 0269/1853] DOC fix after bad merge - remove duplicate entries in changelog --- docs/news.rst | 72 +++++++++++++-------------------------------------- 1 file changed, 18 insertions(+), 54 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index a4f07efad..4711d2f35 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,7 +26,6 @@ Highlights: * various bug fixes, small new features and usability improvements across the codebase. - Selector API changes ~~~~~~~~~~~~~~~~~~~~ @@ -82,6 +81,8 @@ New extensibility features * new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows to select sitemap entries based on their attributes in SitemapSpider subclasses (:issue:`3512`). +* Lazy loading of Downloader Handlers is now optional; this enables better + initialization error handling in custom Downloader Handlers (:issue:`3394`). New FilePipeline and MediaPipeline features ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -104,34 +105,20 @@ New FilePipeline and MediaPipeline features * Fixed errback handling in contracts, e.g. for cases where a contract is executed for URL which returns non-200 response (:issue:`3371`). -Usability and other improvements, cleanups -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Usability improvements +~~~~~~~~~~~~~~~~~~~~~~ -* All Scrapy tests now pass on Windows; Scrapy testing suite is executed - in a Windows environment on CI (:issue:`3315`). -* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). -* Lazy loading of Downloader Handlers is now optional; this enables better - initialization error handling in custom Downloader Handlers (:issue:`3394`). -* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, - :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) -* better error message when an exporter is disabled (:issue:`3358`); -* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" - optional arguments (:issue:`3231`). * more stats for RobotsTxtMiddleware (:issue:`3100`) * INFO log level is used to show telnet host/port (:issue:`3115`) * a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) * better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) * non-zero exit code is returned from Scrapy commands when error happens - on spider inititalization (:issue:`3226`); -* link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); + on spider inititalization (:issue:`3226`) +* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); "flv" is added to common video extensions (:issue:`3165`) +* better error message when an exporter is disabled (:issue:`3358`); * `scrapy shell --help` mentions syntax required for local files (``./file.html``) - :issue:`3496`. -* additional files are included to sdist (:issue:`3495`); -* code style fixes (:issue:`3405`, :issue:`3304`); -* unneeded .strip() call is removed (:issue:`3519`); -* collections.deque is used to store MiddlewareManager methods instead - of a list (:issue:`3476`) * Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) Bug fixes @@ -195,44 +182,21 @@ Other deprecation removals: * Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes are removed (:issue:`3327`, :issue:`3359`). -Internal improvements -~~~~~~~~~~~~~~~~~~~~~ +Other improvements, cleanups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows - to access e.g. settings or a spider from a dupefilter. -* :signal:`item_error` is fired when an error happens in a pipeline - (:issue:`3256`); -* :signal:`request_reached_downloader` is fired when Downloader gets - a new Request; this signal can be useful e.g. for custom Schedulers - (:issue:`3393`). +* All Scrapy tests now pass on Windows; Scrapy testing suite is executed + in a Windows environment on CI (:issue:`3315`). +* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`). +* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`, + :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`) * ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name" optional arguments (:issue:`3231`). - -Usability improvements -~~~~~~~~~~~~~~~~~~~~~~ - -* more stats for RobotsTxtMiddleware (:issue:`3100`) -* INFO log level is used to show telnet host/port (:issue:`3115`) -* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) -* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) -* non-zero exit code is returned from Scrapy commands when error happens - on spider inititalization (:issue:`3226`) -* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); - "flv" is added to common video extensions (:issue:`3165`) - -Bug fixes -~~~~~~~~~ -* proper handling of pickling errors in Python 3 when serializing objects - for disk queues (:issue:`3082`) -* flags are now preserved when copying Requests (:issue:`3342`); -* FormRequest.from_response clickdata shouldn't ignore elements with - ``input[type=image]`` (:issue:`3153`). -* FormRequest.from_response should preserve duplicate keys (:issue:`3247`) - -Cleanups -~~~~~~~~ * additional files are included to sdist (:issue:`3495`); -* code style fixes (:issue:`3405`, :issue:`3304`) +* code style fixes (:issue:`3405`, :issue:`3304`); +* unneeded .strip() call is removed (:issue:`3519`); +* collections.deque is used to store MiddlewareManager methods instead + of a list (:issue:`3476`) .. _release-1.5.2: From 91791cd329936ee6ac53523460f9b72c20c66afb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 30 Jan 2019 17:53:58 +0500 Subject: [PATCH 0270/1853] DOC final changelog cleanups --- docs/news.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 4711d2f35..543901809 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -63,7 +63,8 @@ Telnet console ~~~~~~~~~~~~~~ **Backwards incompatible**: Scrapy's telnet console now requires username -and password. See :ref:`topics-telnetconsole` for more details. +and password. See :ref:`topics-telnetconsole` for more details. This change +fixes a **security issue**; see :ref:`release-1.5.2` release notes for details. New extensibility features ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -117,7 +118,7 @@ Usability improvements * Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); "flv" is added to common video extensions (:issue:`3165`) * better error message when an exporter is disabled (:issue:`3358`); -* `scrapy shell --help` mentions syntax required for local files +* ``scrapy shell --help`` mentions syntax required for local files (``./file.html``) - :issue:`3496`. * Referer header value is added to RFPDupeFilter log messages (:issue:`3588`) From b8594353d03be5574f51766c35566b713584302b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 30 Jan 2019 18:00:40 -0300 Subject: [PATCH 0271/1853] =?UTF-8?q?Bump=20version:=201.5.0=20=E2=86=92?= =?UTF-8?q?=201.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6e7be142e..8cecb7ad4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.5.0 +current_version = 1.6.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index bc80560fa..dc1e644a1 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.5.0 +1.6.0 From 88326cd8be7f9c9f09924144a4d3a9666fdcf0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 31 Jan 2019 01:16:28 -0300 Subject: [PATCH 0272/1853] Set release date to 1.6.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 543901809..668473887 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-1.6.0: -Scrapy 1.6.0 (unreleased) +Scrapy 1.6.0 (2019-01-30) ------------------------- Highlights: From 65d631329a1434ec013f24341e4b8520241aec70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 31 Jan 2019 01:28:53 -0300 Subject: [PATCH 0273/1853] Be consistent with domain used for links to documentation website --- CONTRIBUTING.md | 2 +- INSTALL | 2 +- README.rst | 8 ++++---- docs/contributing.rst | 2 +- docs/topics/selectors.rst | 4 ++-- scrapy/extensions/telnet.py | 2 +- scrapy/templates/project/module/items.py.tmpl | 2 +- .../project/module/middlewares.py.tmpl | 2 +- .../project/module/pipelines.py.tmpl | 2 +- .../templates/project/module/settings.py.tmpl | 20 +++++++++---------- sep/sep-001.rst | 2 +- sep/sep-006.rst | 4 ++-- tests/__init__.py | 2 +- 13 files changed, 27 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a11b05d2..a05d07aee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ The guidelines for contributing are available here: -https://doc.scrapy.org/en/master/contributing.html +https://docs.scrapy.org/en/master/contributing.html Please do not abuse the issue tracker for support questions. If your issue topic can be rephrased to "How to ...?", please use the diff --git a/INSTALL b/INSTALL index a3c7899c6..06e812936 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ For information about installing Scrapy see: * docs/intro/install.rst (local file) -* https://doc.scrapy.org/en/latest/intro/install.html (online version) +* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/README.rst b/README.rst index 1361eac26..c28d217ff 100644 --- a/README.rst +++ b/README.rst @@ -51,18 +51,18 @@ The quick way:: pip install scrapy For more details see the install section in the documentation: -https://doc.scrapy.org/en/latest/intro/install.html +https://docs.scrapy.org/en/latest/intro/install.html Documentation ============= -Documentation is available online at https://doc.scrapy.org/ and in the ``docs`` +Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` directory. Releases ======== -You can find release notes at https://doc.scrapy.org/en/latest/news.html +You can find release notes at https://docs.scrapy.org/en/latest/news.html Community (blog, twitter, mail list, IRC) ========================================= @@ -72,7 +72,7 @@ See https://scrapy.org/community/ Contributing ============ -See https://doc.scrapy.org/en/master/contributing.html +See https://docs.scrapy.org/en/master/contributing.html Code of Conduct --------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index 2369c3436..cf27337c8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,7 +7,7 @@ Contributing to Scrapy .. important:: Double check that you are reading the most recent version of this document at - https://doc.scrapy.org/en/master/contributing.html + https://docs.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9dced7473..df1d67ae8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -100,7 +100,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which provides interactive testing) and an example page located in the Scrapy documentation server: - https://doc.scrapy.org/en/latest/_static/selectors-sample1.html + https://docs.scrapy.org/en/latest/_static/selectors-sample1.html .. _topics-selectors-htmlcode: @@ -113,7 +113,7 @@ For the sake of completeness, here's its full HTML code: First, let's open the shell:: - scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html + scrapy shell https://docs.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have the response available as ``response`` shell variable, and its attached selector in ``response.selector`` attribute. diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index dcf73eb88..26b214ee2 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -112,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 7d766f4fc..a12d08414 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your scraped items # # See documentation in: -# https://doc.scrapy.org/en/latest/topics/items.html +# https://docs.scrapy.org/en/latest/topics/items.html import scrapy diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index c5b542bd6..5debe1cd2 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your spider middleware # # See documentation in: -# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index e58dab089..fb641d447 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -3,7 +3,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class ${ProjectName}Pipeline(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a0557473e..cb220eafc 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,9 +5,9 @@ # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # -# https://doc.scrapy.org/en/latest/topics/settings.html -# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html -# https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' @@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) -# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: @@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True #} # Enable or disable spider middlewares -# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares -# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions -# See https://doc.scrapy.org/en/latest/topics/extensions.html +# See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines -# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) -# See https://doc.scrapy.org/en/latest/topics/autothrottle.html +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 @@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) -# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 3766f38fc..2a66f9802 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -61,7 +61,7 @@ ItemForm -------- Pros: -- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html) +- same API used for Items (see https://docs.scrapy.org/en/latest/topics/items.html) - some people consider setitem API more elegant than methods API Cons: diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 7425c0930..366fcf033 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -16,7 +16,7 @@ Motivation ========== When you use Selectors in Scrapy, your final goal is to "extract" the data that -you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html +you've selected, as the [https://docs.scrapy.org/en/latest/topics/selectors.html XPath Selectors documentation] says (bolding by me): When you’re scraping web pages, the most common task you need to perform is @@ -71,5 +71,5 @@ webpage or set of pages. References ========== - 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) + 1. XPath Selectors (https://docs.scrapy.org/topics/selectors.html) 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html) diff --git a/tests/__init__.py b/tests/__init__.py index 55b1ecde8..a54367f8c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ """ tests: this package contains all Scrapy unittests -see https://doc.scrapy.org/en/latest/contributing.html#running-tests +see https://docs.scrapy.org/en/latest/contributing.html#running-tests """ import os From 5a55c4269d3389df4e486657caa58985f8e37e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 31 Jan 2019 17:20:29 -0200 Subject: [PATCH 0274/1853] Adding GCSFeedStorage --- scrapy/extensions/feedexport.py | 26 ++++++++++++++++++ tests/test_feedexport.py | 48 ++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..00d5d8025 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage): key.close() + +class GCSFeedStorage(BlockingFeedStorage): + + project_id = None + bucket_name = None + blob_name = None + + def __init__(self, uri, project_id): + self.project_id = project_id + u = urlparse(uri) + self.bucket_name = u.hostname + self.blob_name = u.path[1:] # remove first "/" + + @classmethod + def from_crawler(cls, crawler, uri): + return cls(uri, crawler.settings['GCS_PROJECT_ID']) + + def _store_in_thread(self, file): + file.seek(0) + from google.cloud.storage import Client + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file) + + class FTPFeedStorage(BlockingFeedStorage): def __init__(self, uri): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..f3c499b3f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( - IFeedStorage, FileFeedStorage, FTPFeedStorage, + IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler @@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(content, expected_content) +class GCSFeedStorageTest(unittest.TestCase): + + @mock.patch('scrapy.conf.settings', + new={'GCS_PROJECT_ID': 'conf_id' }, create=True) + def test_parse_settings(self): + try: + from google.cloud.storage import Client + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123' } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') + assert storage.project_id == '123' + assert storage.bucket_name == 'mybucket' + assert storage.blob_name == 'export.csv' + + @defer.inlineCallbacks + def test_store(self): + try: + from google.cloud.storage import Client, Bucket, Blob + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + uri = 'gcs://mybucket/export.csv' + project_id = 'myproject-123' + with mock.patch('google.cloud.storage.Client') as m: + client_mock = mock.create_autospec(Client) + m.return_value = client_mock + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id) + yield storage.store(f) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with('mybucket') + bucket_mock.blob.assert_called_once_with('export.csv') + blob_mock.upload_from_file.assert_called_once_with(f) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From a4059851e7b6c8d712b2bc73dbff99be2d569d21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 31 Jan 2019 18:29:15 -0200 Subject: [PATCH 0275/1853] Refactoring tests --- scrapy/utils/test.py | 16 ++++++++++++++++ tests/test_feedexport.py | 15 +++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..84eae97a7 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,6 +7,7 @@ import os from importlib import import_module from twisted.trial.unittest import SkipTest +from tests import mock from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore @@ -91,3 +92,18 @@ def assert_samelines(testcase, text1, text2, msg=None): line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + +def mock_google_cloud_storage(): + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Client, Bucket, Blob + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) \ No newline at end of file diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f3c499b3f..6d23c68c3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,7 +24,8 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) -from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler +from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete, + get_crawler, mock_google_cloud_storage) from scrapy.utils.python import to_native_str @@ -207,22 +208,16 @@ class GCSFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): try: - from google.cloud.storage import Client, Bucket, Blob + from google.cloud.storage import Client except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") uri = 'gcs://mybucket/export.csv' project_id = 'myproject-123' - with mock.patch('google.cloud.storage.Client') as m: - client_mock = mock.create_autospec(Client) + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - f = mock.Mock() storage = GCSFeedStorage(uri, project_id) yield storage.store(f) From 38af090f4d6799a0499b116a62c12509f59f561b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 4 Feb 2019 11:17:58 +0100 Subject: [PATCH 0276/1853] Indicate that users must implement their own authentication result check The example of form-based login could lead some users to think its authentication result check was final. See https://stackoverflow.com/a/54410966/939364 This change should make it more obvious that users are expected to implement their own logic to check whether authentication worked or not. --- docs/topics/request-response.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e29914dbf..76360b15f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -489,6 +489,11 @@ method for this job. Here's an example spider which uses it:: import scrapy + def authentication_failed(response): + # TODO: Check the contents of the response and return True if it failed + # or False if it succeeded. + pass + class LoginSpider(scrapy.Spider): name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] @@ -501,8 +506,7 @@ method for this job. Here's an example spider which uses it:: ) def after_login(self, response): - # check login succeed before going on - if "authentication failed" in response.body: + if authentication_failed(response): self.logger.error("Login failed") return From 1bb6c4154c43d03d00147d810147c4f3c807505e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:04:01 -0200 Subject: [PATCH 0277/1853] Turning into instance attributes --- scrapy/extensions/feedexport.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 00d5d8025..a81f44045 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,10 +149,6 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - project_id = None - bucket_name = None - blob_name = None - def __init__(self, uri, project_id): self.project_id = project_id u = urlparse(uri) From fc6809b024dc25663432b6c0c5780021a827ea20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:08:54 -0200 Subject: [PATCH 0278/1853] Add gcs schema to FEED_STORAGES_BASE --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3734a0a58..1a12f35a3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -145,6 +145,7 @@ FEED_STORAGES_BASE = { 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', + 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { From 4a53de165a53433a9a2a8cc4db34e5507c47fbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:09:56 -0200 Subject: [PATCH 0279/1853] Sorted schemas alphabetically --- scrapy/extensions/feedexport.py | 1 - scrapy/settings/default_settings.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a81f44045..8347b42ca 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -146,7 +146,6 @@ class S3FeedStorage(BlockingFeedStorage): key.close() - class GCSFeedStorage(BlockingFeedStorage): def __init__(self, uri, project_id): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1a12f35a3..8769c01ba 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -142,10 +142,10 @@ FEED_STORAGES = {} FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', - 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', - 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 's3': 'scrapy.extensions.feedexport.S3FeedStorage', + 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { From 2bbbd02bda368c2f052f3aa38f99498a632328bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:45:10 -0200 Subject: [PATCH 0280/1853] Adding an option to set ACL while uploading the blob to GCS --- scrapy/extensions/feedexport.py | 11 ++++++++--- scrapy/settings/default_settings.py | 2 ++ tests/test_feedexport.py | 10 ++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8347b42ca..fbbf9bb97 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id): + def __init__(self, uri, project_id, acl): self.project_id = project_id + self.acl = acl u = urlparse(uri) self.bucket_name = u.hostname self.blob_name = u.path[1:] # remove first "/" @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['GCS_PROJECT_ID']) + return cls( + uri, + crawler.settings['GCS_PROJECT_ID'], + crawler.settings['FEED_STORAGE_GCS_ACL'] + ) def _store_in_thread(self, file): file.seek(0) @@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage): client = Client(project=self.project_id) bucket = client.get_bucket(self.bucket_name) blob = bucket.blob(self.blob_name) - blob.upload_from_file(file) + blob.upload_from_file(file, predefined_acl=self.acl) class FTPFeedStorage(BlockingFeedStorage): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8769c01ba..5d2862980 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_GCS_ACL = None + FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6d23c68c3..5cbca6d28 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase): class GCSFeedStorageTest(unittest.TestCase): @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id' }, create=True) + new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True) def test_parse_settings(self): try: from google.cloud.storage import Client except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') assert storage.project_id == '123' + assert storage.acl == 'publicRead' assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' @@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase): uri = 'gcs://mybucket/export.csv' project_id = 'myproject-123' + acl = 'publicRead' (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock f = mock.Mock() - storage = GCSFeedStorage(uri, project_id) + storage = GCSFeedStorage(uri, project_id, acl) yield storage.store(f) f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) client_mock.get_bucket.assert_called_once_with('mybucket') bucket_mock.blob.assert_called_once_with('export.csv') - blob_mock.upload_from_file.assert_called_once_with(f) + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) class StdoutFeedStorageTest(unittest.TestCase): From cb5f800b0f7029d2bf2e09e9bc1391078c133a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 11:26:33 -0200 Subject: [PATCH 0281/1853] Adding documentation about Google Cloud Storage Feed Export --- docs/topics/feed-exports.rst | 22 ++++++++++++++++++++++ docs/topics/settings.rst | 18 ++++++++++++++++++ scrapy/settings/default_settings.py | 2 ++ 3 files changed, 42 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b64dbfbfd..efb63b0ba 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,6 +185,27 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` +.. _topics-feed-storage-gcs: + +Google Cloud Storage (GCS) +-------------------------- + +The feeds are stored on `Google Cloud Storage`_. + + * URI scheme: ``gcs`` + * Example URIs: + + * ``gcs://mybucket/path/to/export.csv`` + + * Required external libraries: `google-cloud-storage `_. + +For more information about authentication, please refer to `Google Cloud documentation `_. + +You can set a *Project ID* and *Access Control List (ACL)* through the following settings: + + * :setting:`FEED_STORAGE_GCS_ACL` + * :setting:`GCS_PROJECT_ID` + .. _topics-feed-storage-stdout: Standard output @@ -366,3 +387,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore +.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..90ae8fd93 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -749,6 +749,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FEED_STORAGE_GCS_ACL + +FEED_STORAGE_GCS_ACL +-------------------- + +The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. +For more information on how to set this value, please refer to `Google Cloud documentation `_. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -786,6 +794,15 @@ Default: ``"anonymous"`` The username to use for FTP connections when there is no ``"ftp_user"`` in ``Request`` meta. +.. setting:: GCS_PROJECT_ID + +GCS_PROJECT_ID +----------------- + +Default: ``None`` + +The Project ID that will be used when storing data on `Google Cloud Storage`_. + .. setting:: ITEM_PIPELINES ITEM_PIPELINES @@ -1371,3 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search +.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5d2862980..c17e94a64 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -168,6 +168,8 @@ FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' FTP_PASSIVE_MODE = True +GCS_PROJECT_ID = None + HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False From 013568097db04396d780d1c91d37027115af7fe2 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 29 Jan 2019 11:10:06 -0300 Subject: [PATCH 0282/1853] add FEED_STORAGE_S3_ACL setting --- scrapy/extensions/feedexport.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..eb0802261 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -118,6 +118,7 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" + self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private') if self.is_botocore: import botocore.session session = botocore.session.get_session() @@ -137,12 +138,13 @@ class S3FeedStorage(BlockingFeedStorage): file.seek(0) if self.is_botocore: self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file) + Bucket=self.bucketname, Key=self.keyname, Body=file, + ACL=self.policy) else: conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - key.set_contents_from_file(file) + key.set_contents_from_file(file, policy=self.policy) key.close() From ad83ffdf1f4d69ffb62b243429e7b59d0930524c Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:32:46 -0200 Subject: [PATCH 0283/1853] refactoring --- scrapy/extensions/feedexport.py | 19 ++++++-- tests/test_feedexport.py | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index eb0802261..ca30322be 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -93,7 +93,7 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None): # BEGIN Backwards compatibility for initialising without keys (and # without using from_crawler) no_defaults = access_key is None and secret_key is None @@ -118,7 +118,7 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" - self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private') + self.acl = acl if self.is_botocore: import botocore.session session = botocore.session.get_session() @@ -132,19 +132,28 @@ class S3FeedStorage(BlockingFeedStorage): @classmethod def from_crawler(cls, crawler, uri): return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], - crawler.settings['AWS_SECRET_ACCESS_KEY']) + crawler.settings['AWS_SECRET_ACCESS_KEY'], + crawler.settings.get('FEED_STORAGE_S3_ACL')) def _store_in_thread(self, file): file.seek(0) if self.is_botocore: + kwargs = dict() + if self.acl: + kwargs.update(dict(ACL=self.acl)) + self.s3_client.put_object( Bucket=self.bucketname, Key=self.keyname, Body=file, - ACL=self.policy) + **kwargs) else: conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - key.set_contents_from_file(file, policy=self.policy) + kwargs = dict() + if self.acl: + kwargs.update(dict(policy=self.acl)) + + key.set_contents_from_file(file, **kwargs) key.close() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..b07635cb0 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,6 +18,7 @@ from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri +import botocore.client import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( @@ -186,6 +187,89 @@ class S3FeedStorageTest(unittest.TestCase): content = get_s3_content_and_delete(u.hostname, u.path[1:]) self.assertEqual(content, expected_content) + def test_init_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_init_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + def test_from_crawler_without_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + def test_from_crawler_with_acl(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_STORAGE_S3_ACL': 'custom-acl', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + def test_store_in_thread_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + storage._store_in_thread(BytesIO(b'test file')) + operation_name, api_params = _make_api_call_mock.call_args[0] + self.assertEqual(operation_name, 'PutObject') + self.assertNotIn('ACL', api_params) + + def test_store_in_thread_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + storage._store_in_thread(BytesIO(b'test file')) + operation_name, api_params = _make_api_call_mock.call_args[0] + self.assertEqual(operation_name, 'PutObject') + self.assertEqual(api_params.get('ACL'), 'custom-acl') + class StdoutFeedStorageTest(unittest.TestCase): From 126207fb7bca21d3d95ed9c66028e82771180370 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:38:17 -0200 Subject: [PATCH 0284/1853] PEP8: use short name for mock method --- tests/test_feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b07635cb0..bfac06efc 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -247,9 +247,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) - with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + with mock.patch('botocore.client.BaseClient._make_api_call') as m: storage._store_in_thread(BytesIO(b'test file')) - operation_name, api_params = _make_api_call_mock.call_args[0] + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) @@ -264,9 +264,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock: + with mock.patch('botocore.client.BaseClient._make_api_call') as m: storage._store_in_thread(BytesIO(b'test file')) - operation_name, api_params = _make_api_call_mock.call_args[0] + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') From e0f34be383e361c75b22da59c97dee1db189937e Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:50:19 -0200 Subject: [PATCH 0285/1853] update docs --- docs/topics/feed-exports.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b64dbfbfd..661751ed9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,6 +185,10 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` +You can also define a custom ACL for exported objects using this setting: + + * :setting:`FEED_STORAGE_S3_ACL` + .. _topics-feed-storage-stdout: Standard output @@ -205,6 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_URI` (mandatory) * :setting:`FEED_FORMAT` * :setting:`FEED_STORAGES` + * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` @@ -302,11 +307,22 @@ Default: ``{}`` A dict containing additional feed storage backends supported by your project. The keys are URI schemes and the values are paths to storage classes. +.. setting:: FEED_STORAGE_S3_ACL + +FEED_STORAGE_S3_ACL +------------------- + +Default: ``None`` + +A string containing a custom ACL for feeds exported to Amazon S3 by your project. + +For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs. + .. setting:: FEED_STORAGES_BASE FEED_STORAGES_BASE ------------------ - +` Default:: { @@ -366,3 +382,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore +.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl From 7b83ed7c5e1fcd81baf50db3a76f10ade7aa226e Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:52:24 -0200 Subject: [PATCH 0286/1853] remove typo --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 661751ed9..25979dfef 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -322,7 +322,7 @@ For a complete list of available values, access the `Canned ACL`_ section on Ama FEED_STORAGES_BASE ------------------ -` + Default:: { From e25b9a2323c169a4032ff07f912299b32de4b2e0 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 6 Feb 2019 18:52:39 -0200 Subject: [PATCH 0287/1853] calling it feeds instead of objects --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 25979dfef..dee0c3ffa 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,7 +185,7 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` -You can also define a custom ACL for exported objects using this setting: +You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` From dbeb088eea1713ac43f3d23579c36ece5f67563f Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 7 Feb 2019 09:29:16 -0200 Subject: [PATCH 0288/1853] trying to fix jessie testenv by adding botocore to requirements and fixing its version --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 0c0f8f7b7..f2f3e1293 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,7 @@ deps = lxml==3.4.0 Twisted==14.0.2 boto==2.34.0 + botocore==1.12.89 Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 From 079af889e7d010a79640e6874c3e6dc394b936ae Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 7 Feb 2019 10:42:59 -0200 Subject: [PATCH 0289/1853] also testing without botocore --- tests/test_feedexport.py | 53 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index bfac06efc..520ca4a8f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -237,7 +237,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - def test_store_in_thread_without_acl(self): + def test_store_in_thread_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -253,7 +253,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) - def test_store_in_thread_with_acl(self): + def test_store_in_thread_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -270,6 +270,55 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') + def test_store_in_thread_not_botocore_without_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, None) + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + storage._store_in_thread(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertNotIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + + def test_store_in_thread_not_botocore_with_acl(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.acl, 'custom-acl') + + storage.is_botocore = False + storage.connect_s3 = mock.MagicMock() + self.assertFalse(storage.is_botocore) + + storage._store_in_thread(BytesIO(b'test file')) + + conn = storage.connect_s3(*storage.connect_s3.call_args) + bucket = conn.get_bucket(*conn.get_bucket.call_args) + key = bucket.new_key(*bucket.new_key.call_args) + self.assertIn( + dict(policy='custom-acl'), + key.set_contents_from_file.call_args + ) + class StdoutFeedStorageTest(unittest.TestCase): From ceae356e62dc2e56465a58b3d9fe00813e289dd6 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 11:47:35 -0200 Subject: [PATCH 0290/1853] add FEED_STORAGE_S3_ACL to default_settings.py file --- scrapy/settings/default_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3734a0a58..776c5af23 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -158,6 +158,8 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_S3_ACL = None + FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' From cfd183a9d19563f09487af942c9a635d665a1905 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 14:49:26 -0200 Subject: [PATCH 0291/1853] no need to use get here since we're defining a default value in default_settings.py --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ca30322be..2b4594ad8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -133,7 +133,7 @@ class S3FeedStorage(BlockingFeedStorage): def from_crawler(cls, crawler, uri): return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], crawler.settings['AWS_SECRET_ACCESS_KEY'], - crawler.settings.get('FEED_STORAGE_S3_ACL')) + crawler.settings['FEED_STORAGE_S3_ACL']) def _store_in_thread(self, file): file.seek(0) From f824f5b2d17b082dac04505ba27afdfa869a11c7 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 15:19:57 -0200 Subject: [PATCH 0292/1853] testing public method store instead of private method _store_in_thread need to mock deferToThread function --- tests/test_feedexport.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 520ca4a8f..e8c32ea43 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -237,7 +237,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - def test_store_in_thread_botocore_without_acl(self): + def test_store_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -247,13 +247,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + with mock.patch('botocore.client.BaseClient._make_api_call') as m: - storage._store_in_thread(BytesIO(b'test file')) + with mock.patch('twisted.internet.threads.deferToThread', + new=_defer): + storage.store(BytesIO(b'test file')) + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) - def test_store_in_thread_botocore_with_acl(self): + def test_store_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -264,13 +270,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + with mock.patch('botocore.client.BaseClient._make_api_call') as m: - storage._store_in_thread(BytesIO(b'test file')) + with mock.patch('twisted.internet.threads.deferToThread', + new=_defer): + storage.store(BytesIO(b'test file')) + operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') - def test_store_in_thread_not_botocore_without_acl(self): + def test_store_not_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -284,7 +296,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - storage._store_in_thread(BytesIO(b'test file')) + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + + with mock.patch('twisted.internet.threads.deferToThread', new=_defer): + storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) @@ -294,7 +310,7 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) - def test_store_in_thread_not_botocore_with_acl(self): + def test_store_not_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -309,7 +325,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - storage._store_in_thread(BytesIO(b'test file')) + def _defer(f, *args, **kwargs): + return f(*args, **kwargs) + + with mock.patch('twisted.internet.threads.deferToThread', new=_defer): + storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) From 1eac2a163c2c734594d4f1e7e026eab309b2b0b5 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 8 Feb 2019 16:50:39 -0200 Subject: [PATCH 0293/1853] simplifying how we deal with threads.deferToThread calls --- tests/test_feedexport.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e8c32ea43..0f31ef00e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -237,6 +237,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -247,18 +248,14 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - with mock.patch('twisted.internet.threads.deferToThread', - new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertNotIn('ACL', api_params) + @defer.inlineCallbacks def test_store_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -270,18 +267,14 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - with mock.patch('twisted.internet.threads.deferToThread', - new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) operation_name, api_params = m.call_args[0] self.assertEqual(operation_name, 'PutObject') self.assertEqual(api_params.get('ACL'), 'custom-acl') + @defer.inlineCallbacks def test_store_not_botocore_without_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -296,11 +289,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - - with mock.patch('twisted.internet.threads.deferToThread', new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) @@ -310,6 +299,7 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) + @defer.inlineCallbacks def test_store_not_botocore_with_acl(self): storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -325,11 +315,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.connect_s3 = mock.MagicMock() self.assertFalse(storage.is_botocore) - def _defer(f, *args, **kwargs): - return f(*args, **kwargs) - - with mock.patch('twisted.internet.threads.deferToThread', new=_defer): - storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b'test file')) conn = storage.connect_s3(*storage.connect_s3.call_args) bucket = conn.get_bucket(*conn.get_bucket.call_args) From 03e61b9908733f085d87da6bd29152389961b81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Feb 2019 13:50:01 +0100 Subject: [PATCH 0294/1853] Check that spidercls arguments in scrapy.crawler classes are not spider objects --- scrapy/crawler.py | 13 +++++++++++++ tests/test_crawler.py | 13 +++++++++++++ tests/test_downloadermiddleware_httpproxy.py | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04aee18ed..ee00d27b4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -7,6 +7,7 @@ import sys from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement +from scrapy import Spider from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader @@ -27,6 +28,10 @@ logger = logging.getLogger(__name__) class Crawler(object): def __init__(self, spidercls, settings=None): + if isinstance(spidercls, Spider): + raise ValueError( + 'The spidercls argument must be a class, not an object') + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -168,6 +173,10 @@ class CrawlerRunner(object): :param dict kwargs: keyword arguments to initialize the spider """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + 'The crawler_or_spidercls argument cannot be a spider object, ' + 'it must be a spider class (or a Crawler object)') crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) @@ -195,6 +204,10 @@ class CrawlerRunner(object): a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + 'The crawler_or_spidercls argument cannot be a spider object, ' + 'it must be a spider class (or a Crawler object)') if isinstance(crawler_or_spidercls, Crawler): return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 268948a70..37cea3ad3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import warnings from twisted.internet import defer from twisted.trial import unittest +from pytest import raises import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess @@ -66,6 +67,10 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') + def test_crawler_rejects_spider_objects(self): + with raises(ValueError): + Crawler(DefaultSpider()) + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -177,6 +182,14 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertEqual(len(w), 1) self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message)) + def test_crawl_rejects_spider_objects(self): + with raises(ValueError): + CrawlerRunner().crawl(DefaultSpider()) + + def test_create_crawler_rejects_spider_objects(self): + with raises(ValueError): + CrawlerRunner().create_crawler(DefaultSpider()) + class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 537126613..30920b2da 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -25,7 +25,7 @@ class TestHttpProxyMiddleware(TestCase): def test_not_enabled(self): settings = Settings({'HTTPPROXY_ENABLED': False}) - crawler = Crawler(spider, settings) + crawler = Crawler(Spider, settings) self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) def test_no_environment_proxies(self): From 7c9f0bd86c5f02ea803fa6bf1242d34d9c9f47d5 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 12 Feb 2019 12:19:30 -0200 Subject: [PATCH 0295/1853] using named params with optional amazon s3 params --- scrapy/extensions/feedexport.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2b4594ad8..f6bc460ea 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -131,9 +131,12 @@ class S3FeedStorage(BlockingFeedStorage): @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'], - crawler.settings['AWS_SECRET_ACCESS_KEY'], - crawler.settings['FEED_STORAGE_S3_ACL']) + return cls( + uri=uri, + access_key=crawler.settings['AWS_ACCESS_KEY_ID'], + secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + acl=crawler.settings['FEED_STORAGE_S3_ACL'] + ) def _store_in_thread(self, file): file.seek(0) From c2dede27bd56bd783c45fb7302ca06b7c2c025c0 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 12 Feb 2019 12:22:05 -0200 Subject: [PATCH 0296/1853] reduce code with simple ternary operator --- scrapy/extensions/feedexport.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index f6bc460ea..40f985f19 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -141,10 +141,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) if self.is_botocore: - kwargs = dict() - if self.acl: - kwargs.update(dict(ACL=self.acl)) - + kwargs = {'ACL': self.acl} if self.acl else {} self.s3_client.put_object( Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs) @@ -152,10 +149,7 @@ class S3FeedStorage(BlockingFeedStorage): conn = self.connect_s3(self.access_key, self.secret_key) bucket = conn.get_bucket(self.bucketname, validate=False) key = bucket.new_key(self.keyname) - kwargs = dict() - if self.acl: - kwargs.update(dict(policy=self.acl)) - + kwargs = {'policy': self.acl} if self.acl else {} key.set_contents_from_file(file, **kwargs) key.close() From 984e706fd2e06457bcdd1226366d08950ac101b0 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 12 Feb 2019 12:26:57 -0200 Subject: [PATCH 0297/1853] using blank string instead of None as default value as proposed by @kmike --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dee0c3ffa..cf70b8aca 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -312,7 +312,7 @@ The keys are URI schemes and the values are paths to storage classes. FEED_STORAGE_S3_ACL ------------------- -Default: ``None`` +Default: ``''`` (empty string) A string containing a custom ACL for feeds exported to Amazon S3 by your project. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 40f985f19..975fa1229 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -135,7 +135,7 @@ class S3FeedStorage(BlockingFeedStorage): uri=uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None ) def _store_in_thread(self, file): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 776c5af23..a800d39ab 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -158,7 +158,7 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 -FEED_STORAGE_S3_ACL = None +FEED_STORAGE_S3_ACL = '' FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' From 04ccf79e38561a2175c18440b3b4a53ba2f4992f Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Wed, 13 Feb 2019 15:39:45 +0000 Subject: [PATCH 0298/1853] A different S3 Endpoint URL is now possible when uploading images --- scrapy/pipelines/images.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 95323c613..8338a6281 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -89,6 +89,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs'] From 50bf4c60c480a276651ff703cf9fed8e7f981d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Feb 2019 17:39:20 +0100 Subject: [PATCH 0299/1853] Document that the main entry point of downloader and spider middlewares is from_crawler() --- docs/topics/downloader-middleware.rst | 8 ++++++-- docs/topics/spider-middleware.rst | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..18a0639ce 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -55,8 +55,12 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each middleware component is a Python class that defines one or -more of the following methods: +Each downloader middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.downloadermiddlewares diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..62b5ca0e8 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -57,8 +57,12 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each middleware component is a Python class that defines one or more of the -following methods: +Each spider middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.spidermiddlewares From 430e9392483b3992c16bd0314f1bcaed91a9d392 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Wed, 13 Feb 2019 19:59:40 +0000 Subject: [PATCH 0300/1853] Added missing AWS Settings for ImagesPipeline --- scrapy/pipelines/images.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8338a6281..a1457c7e9 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -90,6 +90,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] gcs_store = cls.STORE_SCHEMES['gs'] From b4d132b9f0824263d83331d0b36870f6f64918e4 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 13 Feb 2019 19:21:14 -0200 Subject: [PATCH 0301/1853] setting botocore version as described in debian jessie website https://packages.debian.org/en/jessie/python-botocore --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f2f3e1293..584da2dcd 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ deps = lxml==3.4.0 Twisted==14.0.2 boto==2.34.0 - botocore==1.12.89 + botocore==0.62 Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 From dc0b643832e9f3400e432c2ef7a34e6c75ac8366 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 13 Feb 2019 19:44:50 -0200 Subject: [PATCH 0302/1853] refactoring tests to avoid mocking private method --- tests/test_feedexport.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 0f31ef00e..c103593f9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -248,12 +248,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - yield storage.store(BytesIO(b'test file')) - - operation_name, api_params = m.call_args[0] - self.assertEqual(operation_name, 'PutObject') - self.assertNotIn('ACL', api_params) + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1]) @defer.inlineCallbacks def test_store_botocore_with_acl(self): @@ -267,12 +264,12 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') - with mock.patch('botocore.client.BaseClient._make_api_call') as m: - yield storage.store(BytesIO(b'test file')) - - operation_name, api_params = m.call_args[0] - self.assertEqual(operation_name, 'PutObject') - self.assertEqual(api_params.get('ACL'), 'custom-acl') + storage.s3_client = mock.MagicMock() + yield storage.store(BytesIO(b'test file')) + self.assertEqual( + storage.s3_client.put_object.call_args[1].get('ACL'), + 'custom-acl' + ) @defer.inlineCallbacks def test_store_not_botocore_without_acl(self): From ea8be627d15aa6fe1beaf50fff666cbeb161d94d Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 13 Feb 2019 19:53:10 -0200 Subject: [PATCH 0303/1853] botocore is not supported on debian jessie --- tests/test_feedexport.py | 7 ++++++- tox.ini | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c103593f9..2bf57e278 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,7 +18,6 @@ from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri -import botocore.client import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( @@ -239,6 +238,9 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): + if os.getenv('TOX_ENV_NAME') == 'jessie': + raise unittest.SkipTest('botocore is not supported on jessie') + storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -254,6 +256,9 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): + if os.getenv('TOX_ENV_NAME') == 'jessie': + raise unittest.SkipTest('botocore is not supported on jessie') + storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', diff --git a/tox.ini b/tox.ini index 584da2dcd..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,6 @@ deps = lxml==3.4.0 Twisted==14.0.2 boto==2.34.0 - botocore==0.62 Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 From 9b8ba4c383df0f3029d1b07ab9647a7d902600f4 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 14 Feb 2019 16:20:56 -0200 Subject: [PATCH 0304/1853] try to import botocore before runing some tests --- tests/test_feedexport.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2bf57e278..3ff79c912 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -238,8 +238,10 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): - if os.getenv('TOX_ENV_NAME') == 'jessie': - raise unittest.SkipTest('botocore is not supported on jessie') + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') storage = S3FeedStorage( 's3://mybucket/export.csv', @@ -256,8 +258,10 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): - if os.getenv('TOX_ENV_NAME') == 'jessie': - raise unittest.SkipTest('botocore is not supported on jessie') + try: + import botocore + except ImportError: + raise unittest.SkipTest('botocore is required') storage = S3FeedStorage( 's3://mybucket/export.csv', From b02d26fae8892775ad6ef306d80b02e6bc69d12e Mon Sep 17 00:00:00 2001 From: John de la Garza Date: Fri, 15 Feb 2019 16:54:19 -0800 Subject: [PATCH 0306/1853] rel_has_nofollow: remove redundant if statement --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ccfdcd72..6de36d45c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -116,7 +116,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return True if rel is not None and 'nofollow' in rel.split() else False + return rel is not None and 'nofollow' in rel.split() def create_instance(objcls, settings, crawler, *args, **kwargs): From 0bb3d8ca93cf68e0ef231ee734f8a1a3c4075a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Wed, 27 Feb 2019 18:41:01 -0300 Subject: [PATCH 0307/1853] Updating Google Cloud Storage scheme to gs instead of gcs --- docs/topics/feed-exports.rst | 4 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index efb63b0ba..0957a5997 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -192,10 +192,10 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gcs`` + * URI scheme: ``gs`` * Example URIs: - * ``gcs://mybucket/path/to/export.csv`` + * ``gs://mybucket/path/to/export.csv`` * Required external libraries: `google-cloud-storage `_. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c17e94a64..50fcd1d0a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -143,7 +143,7 @@ FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', - 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 'gs': 'scrapy.extensions.feedexport.GCSFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 5cbca6d28..41df7d7af 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -200,7 +200,7 @@ class GCSFeedStorageTest(unittest.TestCase): settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.project_id == '123' assert storage.acl == 'publicRead' assert storage.bucket_name == 'mybucket' From e3b15252c80ca3d0872f3068c382a0a3e7cc9db6 Mon Sep 17 00:00:00 2001 From: Matthieu Grandrie Date: Thu, 21 Feb 2019 17:19:58 +0100 Subject: [PATCH 0308/1853] New constructor arg *restrict_text* for FilteringLinkExtractor. Same as allow and deny args, it holds a string, a regex or an iterable of. Links whose text don't match one of the regex are filtered out. DOC restrict_text in LxmlLinkExtractor --- docs/topics/link-extractors.rst | 6 ++++++ scrapy/linkextractors/__init__.py | 6 +++++- scrapy/linkextractors/lxmlhtml.py | 9 +++++---- scrapy/linkextractors/sgml.py | 13 +++++++------ tests/test_linkextractors.py | 24 ++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f40a36d31..713a94e10 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -93,6 +93,12 @@ LxmlLinkExtractor Has the same behaviour as ``restrict_xpaths``. :type restrict_css: str or list + :param restrict_text: a single regular expression (or list of regular expressions) + that the link's text must match in order to be extracted. If not + given (or empty), it will match all links. If a list of regular expressions is + given, the link will be extracted if it matches at least one. + :type restrict_text: a regular expression (or list of) + :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. :type tags: str or list diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 97e8c0af1..ebf3cd7d8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -50,7 +50,7 @@ class FilteringLinkExtractor(object): _csstranslator = HTMLTranslator() def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css): + restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): self.link_extractor = link_extractor @@ -70,6 +70,8 @@ class FilteringLinkExtractor(object): if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] def _link_allowed(self, link): if not _is_valid_url(link.url): @@ -85,6 +87,8 @@ class FilteringLinkExtractor(object): return False if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False return True def matches(self, url): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index a7092f9b8..8f6f93a44 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -97,7 +97,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs @@ -111,9 +111,10 @@ class LxmlLinkExtractor(FilteringLinkExtractor): ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = get_base_url(response) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 5fa6b771c..8940a4d77 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -113,7 +113,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=()): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -127,13 +127,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip, - canonicalized=canonicalize) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = None diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 903032b52..c9cd629f4 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -479,6 +479,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_link_restrict_text(self): + html = b""" + Pic of a cat + Pic of a dog + Pic of a cow + """ + response = HtmlResponse("http://example.org/index.html", body=html) + # Simple text inclusion test + lx = self.extractor_cls(restrict_text='dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Unique regex test + lx = self.extractor_cls(restrict_text=r'of.*dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Multiple regex test + lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() From 858f5be74728209d8ef71794296814abca4c1c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2019 16:10:23 +0100 Subject: [PATCH 0309/1853] =?UTF-8?q?backwards=20=E2=86=92=20backward=20(a?= =?UTF-8?q?dj.)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 64 ++++++++++++------------- docs/topics/request-response.rst | 2 +- docs/topics/spiders.rst | 2 +- docs/versioning.rst | 2 +- scrapy/cmdline.py | 4 +- scrapy/conf.py | 2 +- scrapy/core/downloader/handlers/http.py | 2 +- scrapy/extensions/feedexport.py | 4 +- scrapy/log.py | 2 +- scrapy/signals.py | 2 +- scrapy/utils/conf.py | 4 +- sep/sep-018.rst | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_feedexport.py | 2 +- tests/test_utils_conf.py | 2 +- 15 files changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 668473887..7ac1664fe 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -53,7 +53,7 @@ case of Scrapy spiders: callbacks are usually called several times, on different pages. If you're using custom ``Selector`` or ``SelectorList`` subclasses, -a **backwards incompatible** change in parsel may affect your code. +a **backward incompatible** change in parsel may affect your code. See `parsel changelog`_ for a detailed description, as well as for the full list of improvements. @@ -62,7 +62,7 @@ full list of improvements. Telnet console ~~~~~~~~~~~~~~ -**Backwards incompatible**: Scrapy's telnet console now requires username +**Backward incompatible**: Scrapy's telnet console now requires username and password. See :ref:`topics-telnetconsole` for more details. This change fixes a **security issue**; see :ref:`release-1.5.2` release notes for details. @@ -209,7 +209,7 @@ Scrapy 1.5.2 (2019-01-22) exploit it from Scrapy, but it is very easy to trick a browser to do so and elevates the risk for local development environment. - *The fix is backwards incompatible*, it enables telnet user-password + *The fix is backward incompatible*, it enables telnet user-password authentication by default with a random generated password. If you can't upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT` out of its default value. @@ -256,15 +256,15 @@ Some highlights: * Better default handling of HTTP 308, 522 and 524 status codes. * Documentation is improved, as usual. -Backwards Incompatible Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backward Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Scrapy 1.5 drops support for Python 3.3. * Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). - **This is technically backwards-incompatible**; override + **This is technically backward-incompatible**; override :setting:`USER_AGENT` if you relied on old value. * Logging of settings overridden by ``custom_settings`` is fixed; - **this is technically backwards-incompatible** because the logger + **this is technically backward-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're parsing Scrapy logs, please update your log parsers (:issue:`1343`). * LinkExtractor now ignores ``m4v`` extension by default, this is change @@ -301,11 +301,11 @@ Bug fixes ~~~~~~~~~ - Fix logging of settings overridden by ``custom_settings``; - **this is technically backwards-incompatible** because the logger + **this is technically backward-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please update your log parsers if needed (:issue:`1343`) - Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). - **This is technically backwards-incompatible**; override + **This is technically backward-incompatible**; override :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, @@ -415,18 +415,18 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting. Enjoy! (Or read on for the rest of changes in this release.) -Deprecations and Backwards Incompatible Changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Deprecations and Backward Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): - **warning, this is technically backwards-incompatible** + **warning, this is technically backward-incompatible** - Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`); - **this is technically backwards-incompatible** so please check if you have + **this is technically backward-incompatible** so please check if you have any non-default ``MEMUSAGE_***`` options set. - ``EDITOR`` environment variable now takes precedence over ``EDITOR`` option defined in settings.py (:issue:`1829`); Scrapy default settings - no longer depend on environment variables. **This is technically a backwards + no longer depend on environment variables. **This is technically a backward incompatible change**. - ``Spider.make_requests_from_url`` is deprecated (:issue:`1728`, fixes :issue:`1495`). @@ -636,10 +636,10 @@ New Features scrapy shell now follow HTTP redirections by default (:issue:`2290`); See :command:`fetch` and :command:`shell` for details. - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; - this is technically **backwards incompatible** so please check your log parsers. + this is technically **backward incompatible** so please check your log parsers. - By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); - this is **backwards incompatible** if you have log parsers expecting the short + this is **backward incompatible** if you have log parsers expecting the short logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` set to ``True``. @@ -750,11 +750,11 @@ Bug fixes ~~~~~~~~~ - DefaultRequestHeaders middleware now runs before UserAgent middleware - (:issue:`2088`). **Warning: this is technically backwards incompatible**, + (:issue:`2088`). **Warning: this is technically backward incompatible**, though we consider this a bug fix. - HTTP cache extension and plugins that use the ``.scrapy`` data directory now work outside projects (:issue:`1581`). **Warning: this is technically - backwards incompatible**, though we consider this a bug fix. + backward incompatible**, though we consider this a bug fix. - ``Selector`` does not allow passing both ``response`` and ``text`` anymore (:issue:`2153`). - Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`). @@ -934,13 +934,13 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Accept XML node names containing dots as valid (:issue:`1533`). - When uploading files or images to S3 (with ``FilesPipeline`` or ``ImagesPipeline``), the default ACL policy is now "private" instead - of "public" **Warning: backwards incompatible!**. + of "public" **Warning: backward incompatible!**. You can use :setting:`FILES_STORE_S3_ACL` to change it. - We've reimplemented ``canonicalize_url()`` for more correct output, especially for URLs with non-ASCII characters (:issue:`1947`). This could change link extractors output compared to previous scrapy versions. This may also invalidate some cache entries you could still have from pre-1.1 runs. - **Warning: backwards incompatible!**. + **Warning: backward incompatible!**. Keep reading for more details on other improvements and bug fixes. @@ -973,7 +973,7 @@ Additional New Features and Enhancements - Support for bpython and configure preferred Python shell via ``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`). - Support URLs without scheme (:issue:`1498`) - **Warning: backwards incompatible!** + **Warning: backward incompatible!** - Bring back support for relative file path (:issue:`1710`, :issue:`1550`). - Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check @@ -1056,7 +1056,7 @@ Bugfixes ~~~~~~~~ - Scrapy does not retry requests that got a ``HTTP 400 Bad Request`` - response anymore (:issue:`1289`). **Warning: backwards incompatible!** + response anymore (:issue:`1289`). **Warning: backward incompatible!** - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). @@ -1646,7 +1646,7 @@ Scrapy 0.24.2 (2014-07-08) Scrapy 0.24.1 (2014-06-27) -------------------------- -- Fix deprecated CrawlerSettings and increase backwards compatibility with +- Fix deprecated CrawlerSettings and increase backward compatibility with .defaults attribute (:commit:`8e3f20a`) @@ -1772,7 +1772,7 @@ Scrapy 0.22.0 (released 2014-01-17) Enhancements ~~~~~~~~~~~~ -- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) +- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) @@ -2092,7 +2092,7 @@ Scrapy 0.16.1 (released 2012-10-26) ----------------------------------- - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`) -- better backwards compatibility for scrapy.conf.settings (:commit:`3403089`) +- better backward compatibility for scrapy.conf.settings (:commit:`3403089`) - extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`) - removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`) - fix dashes under rst headers (:commit:`fa4f7f9`) @@ -2107,7 +2107,7 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` -- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backwards compatibility is kept on the Stats Collector API and signals. +- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. @@ -2259,7 +2259,7 @@ Code rearranged and removed - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) - removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`) - Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`) -- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept. +- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backward compatibility kept. Scrapy 0.12 ----------- @@ -2356,11 +2356,11 @@ API changes - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` - default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201) - changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - moved ``scrapy.core.signals`` module to ``scrapy.signals`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions`` - - backwards compatibility kept (with deprecation warning) + - backward compatibility kept (with deprecation warning) - added ``handles_request()`` class method to ``BaseSpider`` - dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead) - dropped ``component`` argument of ``scrapy.log.msg()`` function @@ -2431,8 +2431,8 @@ New features - Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`) - Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`) -Backwards-incompatible changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`) - Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 76360b15f..4511f3469 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -728,7 +728,7 @@ TextResponse objects .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is - kept for backwards compatibility; please prefer ``response.text``. + kept for backward compatibility; please prefer ``response.text``. HtmlResponse objects diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..e1d36aa24 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -190,7 +190,7 @@ scrapy.Spider .. method:: log(message, [level, component]) Wrapper that sends a log message through the Spider's :attr:`logger`, - kept for backwards compatibility. For more information see + kept for backward compatibility. For more information see :ref:`topics-logging-from-spiders`. .. method:: closed(reason) diff --git a/docs/versioning.rst b/docs/versioning.rst index 0421ba544..227085f02 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very large changes. * *B* is the release number. This will include many changes including features - and things that possibly break backwards compatibility, although we strive to + and things that possibly break backward compatibility, although we strive to keep theses cases at a minimum. * *C* is the bugfix release number. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index dc6b59fe0..fa2506eb0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -99,7 +99,7 @@ def execute(argv=None, settings=None): if argv is None: argv = sys.argv - # --- backwards compatibility for scrapy.conf.settings singleton --- + # --- backward compatibility for scrapy.conf.settings singleton --- if settings is None and 'scrapy.conf' in sys.modules: from scrapy import conf if hasattr(conf, 'settings'): @@ -116,7 +116,7 @@ def execute(argv=None, settings=None): settings['EDITOR'] = editor check_deprecated_settings(settings) - # --- backwards compatibility for scrapy.conf.settings singleton --- + # --- backward compatibility for scrapy.conf.settings singleton --- import warnings from scrapy.exceptions import ScrapyDeprecationWarning with warnings.catch_warnings(): diff --git a/scrapy/conf.py b/scrapy/conf.py index 23efc6ffd..6c40edcdd 100644 --- a/scrapy/conf.py +++ b/scrapy/conf.py @@ -1,4 +1,4 @@ -# This module is kept for backwards compatibility, so users can import +# This module is kept for backward compatibility, so users can import # scrapy.conf.settings and get the settings they expect import sys diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index e4a7d8564..e76823623 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -3,7 +3,7 @@ from .http10 import HTTP10DownloadHandler from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler -# backwards compatibility +# backward compatibility class HttpDownloadHandler(HTTP10DownloadHandler): def __init__(self, *args, **kwargs): diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..3b4d809e8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,7 +94,7 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None): - # BEGIN Backwards compatibility for initialising without keys (and + # BEGIN Backward compatibility for initialising without keys (and # without using from_crawler) no_defaults = access_key is None and secret_key is None if no_defaults: @@ -111,7 +111,7 @@ class S3FeedStorage(BlockingFeedStorage): ) access_key = settings['AWS_ACCESS_KEY_ID'] secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backwards compatibility + # END Backward compatibility u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/scrapy/log.py b/scrapy/log.py index 719fceaad..777bd6dc4 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -17,7 +17,7 @@ warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on " ScrapyDeprecationWarning, stacklevel=2) -# Imports and level_names variable kept for backwards-compatibility +# Imports and level_names variable kept for backward-compatibility DEBUG = logging.DEBUG INFO = logging.INFO diff --git a/scrapy/signals.py b/scrapy/signals.py index c0e4bb74e..6b9125302 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -20,7 +20,7 @@ item_scraped = object() item_dropped = object() item_error = object() -# for backwards compatibility +# for backward compatibility stats_spider_opened = spider_opened stats_spider_closing = spider_closed stats_spider_closed = spider_closed diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 435e9a6b3..fbd297340 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -42,14 +42,14 @@ def build_component_list(compdict, custom=None, convert=update_classpath): raise ValueError('Invalid value {} for component {}, please provide ' \ 'a real number or None instead'.format(value, name)) - # BEGIN Backwards compatibility for old (base, custom) call signature + # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) return type(custom)(convert(c) for c in custom) if custom is not None: compdict.update(custom) - # END Backwards compatibility + # END Backward compatibility _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index aca7ac342..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -211,7 +211,7 @@ spider methods on each event such as: - call additional spider middlewares defined in the ``Spider.middlewares`` attribute - call ``Spider.next_request()`` and ``Spider.start_requests()`` on - ``next_request()`` middleware method (this would implicitly support backwards + ``next_request()`` middleware method (this would implicitly support backward compatibility) Differences with Spider middleware v1 diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0d0829793..81235a16f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -50,7 +50,7 @@ class DummyDH(object): class DummyLazyDH(object): - # Default is lazy for backwards compatibility + # Default is lazy for backward compatibility def __init__(self, crawler): pass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..b254b9f38 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -161,7 +161,7 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backwards compatibility for initialising without settings + # Backward compatibility for initialising without settings with warnings.catch_warnings(record=True) as w: storage = S3FeedStorage('s3://mybucket/export.csv') self.assertEqual(storage.access_key, 'conf_key') diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f203c32ef..29937c189 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -11,7 +11,7 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ['one', 'four', 'three']) - def test_backwards_compatible_build_dict(self): + def test_backward_compatible_build_dict(self): base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} custom = {'two': None, 'three': 8, 'four': 4} self.assertEqual(build_component_list(base, custom, From 75d6f56c8a731ea4e1c06814a59a0b51741d04a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2019 16:56:58 +0100 Subject: [PATCH 0310/1853] Switch from ` to `` where inline code formatting is desired --- docs/contributing.rst | 2 +- docs/news.rst | 130 +++++++++++++------------- docs/topics/api.rst | 4 +- docs/topics/downloader-middleware.rst | 40 ++++---- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 4 +- docs/topics/jobs.rst | 2 +- docs/topics/loaders.rst | 2 +- docs/topics/logging.rst | 2 +- docs/topics/media-pipeline.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/request-response.rst | 8 +- docs/topics/selectors.rst | 2 +- docs/topics/settings.rst | 4 +- docs/topics/spider-middleware.rst | 4 +- docs/topics/spiders.rst | 6 +- docs/topics/ubuntu.rst | 4 +- scrapy/crawler.py | 10 +- scrapy/logformatter.py | 18 ++-- scrapy/pipelines/files.py | 6 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/python.py | 18 ++-- scrapy/utils/url.py | 8 +- sep/sep-006.rst | 5 +- tests/mocks/dummydbm.py | 2 +- tests/test_command_shell.py | 4 +- 27 files changed, 148 insertions(+), 147 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index cf27337c8..9b508e418 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -55,7 +55,7 @@ guidelines when you're going to report a new bug. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has been discussed there, or if you're not sure if what you're seeing is a bug. - You can also ask in the `#scrapy` IRC channel. + You can also ask in the ``#scrapy`` IRC channel. * write **complete, reproducible, specific bug reports**. The smaller the test case, the better. Remember that other developers won't have your project to diff --git a/docs/news.rst b/docs/news.rst index 668473887..1849a3ca8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -149,7 +149,7 @@ Documentation improvements * improved links to beginner resources in the tutorial (:issue:`3367`, :issue:`3468`); * fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`); -* remove unused `DEPTH_STATS` option from docs (:issue:`3245`); +* remove unused ``DEPTH_STATS`` option from docs (:issue:`3245`); * other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`, :issue:`3605`). @@ -1313,7 +1313,7 @@ Module Relocations There’s been a large rearrangement of modules trying to improve the general structure of Scrapy. Main changes were separating various subpackages into -new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp` +new projects and dissolving both ``scrapy.contrib`` and ``scrapy.contrib_exp`` into top level packages. Backward compatibility was kept among internal relocations, while importing deprecated modules expect warnings indicating their new place. @@ -1344,7 +1344,7 @@ Outsourced packages | | /scrapy-plugins/scrapy-jsonrpc>`_ | +-------------------------------------+-------------------------------------+ -`scrapy.contrib_exp` and `scrapy.contrib` dissolutions +``scrapy.contrib_exp`` and ``scrapy.contrib`` dissolutions +-------------------------------------+-------------------------------------+ | Old location | New location | @@ -1556,7 +1556,7 @@ Code refactoring (:issue:`1078`) - Pydispatch pep8 (:issue:`992`) - Removed unused 'load=False' parameter from walk_modules() (:issue:`871`) -- For consistency, use `job_dir` helper in `SpiderState` extension. +- For consistency, use ``job_dir`` helper in ``SpiderState`` extension. (:issue:`805`) - rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`) @@ -1669,10 +1669,10 @@ Enhancements cache middleware (:issue:`541`, :issue:`500`, :issue:`571`) - Expose current crawler in Scrapy shell (:issue:`557`) - Improve testsuite comparing CSV and XML exporters (:issue:`570`) -- New `offsite/filtered` and `offsite/domains` stats (:issue:`566`) +- New ``offsite/filtered`` and ``offsite/domains`` stats (:issue:`566`) - Support process_links as generator in CrawlSpider (:issue:`555`) - Verbose logging and new stats counters for DupeFilter (:issue:`553`) -- Add a mimetype parameter to `MailSender.send()` (:issue:`602`) +- Add a mimetype parameter to ``MailSender.send()`` (:issue:`602`) - Generalize file pipeline log messages (:issue:`622`) - Replace unencodeable codepoints with html entities in SGMLLinkExtractor (:issue:`565`) - Converted SEP documents to rst format (:issue:`629`, :issue:`630`, @@ -1691,20 +1691,20 @@ Enhancements - Make scrapy.version_info a tuple of integers (:issue:`681`, :issue:`692`) - Infer exporter's output format from filename extensions (:issue:`546`, :issue:`659`, :issue:`760`) -- Support case-insensitive domains in `url_is_from_any_domain()` (:issue:`693`) +- Support case-insensitive domains in ``url_is_from_any_domain()`` (:issue:`693`) - Remove pep8 warnings in project and spider templates (:issue:`698`) -- Tests and docs for `request_fingerprint` function (:issue:`597`) -- Update SEP-19 for GSoC project `per-spider settings` (:issue:`705`) +- Tests and docs for ``request_fingerprint`` function (:issue:`597`) +- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`) - Set exit code to non-zero when contracts fails (:issue:`727`) - Add a setting to control what class is instanciated as Downloader component (:issue:`738`) -- Pass response in `item_dropped` signal (:issue:`724`) -- Improve `scrapy check` contracts command (:issue:`733`, :issue:`752`) -- Document `spider.closed()` shortcut (:issue:`719`) -- Document `request_scheduled` signal (:issue:`746`) +- Pass response in ``item_dropped`` signal (:issue:`724`) +- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`) +- Document ``spider.closed()`` shortcut (:issue:`719`) +- Document ``request_scheduled`` signal (:issue:`746`) - Add a note about reporting security issues (:issue:`697`) - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`) -- Sort spider list output of `scrapy list` command (:issue:`742`) +- Sort spider list output of ``scrapy list`` command (:issue:`742`) - Multiple documentation enhancemens and fixes (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`, :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`, @@ -1772,23 +1772,23 @@ Scrapy 0.22.0 (released 2014-01-17) Enhancements ~~~~~~~~~~~~ -- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) - To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage` +- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) + To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage`` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) - Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`) - Selectors register EXSLT namespaces by default (:issue:`472`) - Unify item loaders similar to selectors renaming (:issue:`461`) -- Make `RFPDupeFilter` class easily subclassable (:issue:`533`) +- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in `get_func_args` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) - Allow running indiviual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) - Improve offsite middleware tests (:issue:`478`) - Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`) -- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`) +- Do not send ``x-gzip`` in default ``Accept-Encoding`` header (:issue:`469`) - Support defining http error handling using settings (:issue:`466`) - Use modern python idioms wherever you find legacies (:issue:`497`) - Improve and correct documentation @@ -1799,14 +1799,14 @@ Fixes ~~~~~ - Update Selector class imports in CrawlSpider template (:issue:`484`) -- Fix unexistent reference to `engine.slots` (:issue:`464`) -- Do not try to call `body_as_unicode()` on a non-TextResponse instance (:issue:`462`) +- Fix unexistent reference to ``engine.slots`` (:issue:`464`) +- Do not try to call ``body_as_unicode()`` on a non-TextResponse instance (:issue:`462`) - Warn when subclassing XPathItemLoader, previously it only warned on instantiation. (:issue:`523`) - Warn when subclassing XPathSelector, previously it only warned on instantiation. (:issue:`537`) - Multiple fixes to memory stats (:issue:`531`, :issue:`530`, :issue:`529`) -- Fix overriding url in `FormRequest.from_response()` (:issue:`507`) +- Fix overriding url in ``FormRequest.from_response()`` (:issue:`507`) - Fix tests runner under pip 1.5 (:issue:`513`) - Fix logging error when spider name is unicode (:issue:`479`) @@ -1833,7 +1833,7 @@ Enhancements (modifying them had been deprecated for a long time) - :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list) - Sitemap spider can fetch alternate URLs (:issue:`360`) -- `Selector.remove_namespaces()` now remove namespaces from element's attributes. (:issue:`416`) +- ``Selector.remove_namespaces()`` now remove namespaces from element's attributes. (:issue:`416`) - Paved the road for Python 3.3+ (:issue:`435`, :issue:`436`, :issue:`431`, :issue:`452`) - New item exporter using native python types with nesting support (:issue:`366`) - Tune HTTP1.1 pool size so it matches concurrency defined by settings (:commit:`b43b5f575`) @@ -1844,13 +1844,13 @@ Enhancements - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`) - Remove multi spider support from multiple core components (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`) -- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages. +- Travis-CI now tests Scrapy changes against development versions of ``w3lib`` and ``queuelib`` python packages. - Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`) - Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`) - Use importlib for parametric imports (:issue:`445`) - Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`) - Bugfix crawling shutdown on SIGINT (:issue:`450`) -- Do not submit `reset` type inputs in FormRequest.from_response (:commit:`b326b87`) +- Do not submit ``reset`` type inputs in FormRequest.from_response (:commit:`b326b87`) - Do not silence download errors when request errback raises an exception (:commit:`684cfc0`) Bugfixes @@ -1865,8 +1865,8 @@ Bugfixes - Improve request-response docs (:issue:`391`) - Improve best practices docs (:issue:`399`, :issue:`400`, :issue:`401`, :issue:`402`) - Improve django integration docs (:issue:`404`) -- Document `bindaddress` request meta (:commit:`37c24e01d7`) -- Improve `Request` class documentation (:issue:`226`) +- Document ``bindaddress`` request meta (:commit:`37c24e01d7`) +- Improve ``Request`` class documentation (:issue:`226`) Other ~~~~~ @@ -1875,7 +1875,7 @@ Other - Add `cssselect`_ python package as install dependency - Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on. - Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support. -- Running test suite now requires `mock` python library (:issue:`390`) +- Running test suite now requires ``mock`` python library (:issue:`390`) Thanks @@ -1929,7 +1929,7 @@ Scrapy 0.18.3 (released 2013-10-03) Scrapy 0.18.2 (released 2013-09-03) ----------------------------------- -- Backport `scrapy check` command fixes and backward compatible multi +- Backport ``scrapy check`` command fixes and backward compatible multi crawler process(:issue:`339`) Scrapy 0.18.1 (released 2013-08-27) @@ -1958,31 +1958,31 @@ Scrapy 0.18.0 (released 2013-08-09) - Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) - Use lxml recover option to parse sitemaps (:issue:`347`) - Bugfix cookie merging by hostname and not by netloc (:issue:`352`) -- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`) -- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`) -- Support `dont_cache` request meta flag (:issue:`19`) -- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`) -- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`) -- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`) +- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`) +- Support xml namespaces using ``iternodes`` parser in ``XMLFeedSpider`` (:issue:`12`) +- Support ``dont_cache`` request meta flag (:issue:`19`) +- Bugfix ``scrapy.utils.gz.gunzip`` broken by changes in python 2.7.4 (:commit:`4dc76e`) +- Bugfix url encoding on ``SgmlLinkExtractor`` (:issue:`24`) +- Bugfix ``TakeFirst`` processor shouldn't discard zero (0) value (:issue:`59`) - Support nested items in xml exporter (:issue:`66`) - Improve cookies handling performance (:issue:`77`) - Log dupe filtered requests once (:issue:`105`) - Split redirection middleware into status and meta based middlewares (:issue:`78`) - Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`) -- Support xpath form selection on `FormRequest.from_response` (:issue:`185`) -- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`) +- Support xpath form selection on ``FormRequest.from_response`` (:issue:`185`) +- Bugfix unicode decoding error on ``SgmlLinkExtractor`` (:issue:`199`) - Bugfix signal dispatching on pypi interpreter (:issue:`205`) - Improve request delay and concurrency handling (:issue:`206`) -- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`) +- Add RFC2616 cache policy to ``HttpCacheMiddleware`` (:issue:`212`) - Allow customization of messages logged by engine (:issue:`214`) -- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`) +- Multiples improvements to ``DjangoItem`` (:issue:`217`, :issue:`218`, :issue:`221`) - Extend Scrapy commands using setuptools entry points (:issue:`260`) -- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`) -- Support `settings.getdict` (:issue:`269`) -- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`) -- Added `Item.copy` (:issue:`290`) +- Allow spider ``allowed_domains`` value to be set/tuple (:issue:`261`) +- Support ``settings.getdict`` (:issue:`269`) +- Simplify internal ``scrapy.core.scraper`` slot handling (:issue:`271`) +- Added ``Item.copy`` (:issue:`290`) - Collect idle downloader slots (:issue:`297`) -- Add `ftp://` scheme downloader handler (:issue:`329`) +- Add ``ftp://`` scheme downloader handler (:issue:`329`) - Added downloader benchmark webserver and spider tools :ref:`benchmarking` - Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on - Add scrapy commands using external libraries (:issue:`260`) @@ -2113,7 +2113,7 @@ Scrapy changes: - dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` -- `lxml` is now the default selectors backend instead of `libxml2` +- ``lxml`` is now the default selectors backend instead of ``libxml2`` - ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_ - removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm`` - SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`) @@ -2206,16 +2206,16 @@ New features and settings - New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`) - Add boto 2.0 support for S3 downloader handler (:rev:`2763`) - Added `marshal`_ to formats supported by feed exports (:rev:`2744`) -- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`) +- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) -- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`) -- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`) -- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) +- Moved spider queues to scrapyd: ``scrapy.spiderqueue`` -> ``scrapyd.spiderqueue`` (:rev:`2708`) +- Moved sqlite utils to scrapyd: ``scrapy.utils.sqlite`` -> ``scrapyd.sqlite`` (:rev:`2781`) +- Real support for returning iterators on ``start_requests()`` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) - Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`) - Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`) - Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`) @@ -2223,19 +2223,19 @@ New features and settings - Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`) - Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`) - Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`) -- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`. +- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an ``IOError``. - Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`) -- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`) +- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and ``-e`` flag to ``genspider`` command that uses it (:rev:`2653`) - Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines. - Added :signal:`spider_error` signal (:rev:`2628`) - Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`) -- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. +- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to ``True``). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. - Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`) - Added new DBM HTTP cache storage backend (:rev:`2576`) - Added ``listjobs.json`` API to Scrapyd (:rev:`2571`) - ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`) - Added namespace support to ``xmliter_lxml`` (:rev:`2552`) -- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`) +- Improved cookies middleware by making ``COOKIES_DEBUG`` nicer and documenting it (:rev:`2579`) - Several improvements to Scrapyd and Link extractors Code rearranged and removed @@ -2249,11 +2249,11 @@ Code rearranged and removed - Reduced Scrapy codebase by striping part of Scrapy code into two new libraries: - `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`) - `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`) -- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`) -- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot +- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`) +- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on github: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) @@ -2289,13 +2289,13 @@ Scrapyd changes - Scrapyd now uses one process per spider - It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default -- There is now a `scrapy server` command to start a Scrapyd server of the current project +- There is now a ``scrapy server`` command to start a Scrapyd server of the current project Changes to settings ~~~~~~~~~~~~~~~~~~~ -- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware -- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire". +- added ``HTTPCACHE_ENABLED`` setting (False by default) to enable HTTP cache middleware +- changed ``HTTPCACHE_EXPIRATION_SECS`` semantics: now zero means "never expire". Deprecated/obsoleted functionality ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2326,17 +2326,17 @@ New features and improvements - Splitted Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) -- Added `dont_redirect` request.meta key for avoiding redirects (#233) -- Added `dont_retry` request.meta key for avoiding retries (#234) +- Added ``dont_redirect`` request.meta key for avoiding redirects (#233) +- Added ``dont_retry`` request.meta key for avoiding retries (#234) Command-line tool changes ~~~~~~~~~~~~~~~~~~~~~~~~~ -- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199) - - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project - - Added `scrapy.bat` script for running more conveniently from Windows +- New ``scrapy`` command which replaces the old ``scrapy-ctl.py`` (#199) + - there is only one global ``scrapy`` command now, instead of one ``scrapy-ctl.py`` per project + - Added ``scrapy.bat`` script for running more conveniently from Windows - Added bash completion to command-line tool (#210) -- Renamed command `start` to `runserver` (#209) +- Renamed command ``start`` to ``runserver`` (#209) API changes ~~~~~~~~~~~ diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 985cc0433..ba832ab5d 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -94,7 +94,7 @@ how you :ref:`configure the downloader middlewares .. method:: crawl(\*args, \**kwargs) Starts the crawler by instantiating its spider class with the given - `args` and `kwargs` arguments, while setting the execution engine in + ``args`` and ``kwargs`` arguments, while setting the execution engine in motion. Returns a deferred that is fired when the crawl is finished. @@ -180,7 +180,7 @@ SpiderLoader API .. method:: load(spider_name) Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name `spider_name` and will raise + loaded spiders for a spider class with name ``spider_name`` and will raise a KeyError if not found. :param spider_name: spider class name diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..e6812eddd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it -in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { @@ -357,7 +357,7 @@ HttpCacheMiddleware .. reqmeta:: dont_cache - You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`. + You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``. .. _httpcache-policy-dummy: @@ -390,17 +390,17 @@ runs to avoid downloading unmodified data (to save bandwidth and speed up crawls what is implemented: -* Do not attempt to store responses/requests with `no-store` cache-control directive set -* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses -* Compute freshness lifetime from `max-age` cache-control directive -* Compute freshness lifetime from `Expires` response header -* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox) -* Compute current age from `Age` response header -* Compute current age from `Date` header -* Revalidate stale responses based on `Last-Modified` response header -* Revalidate stale responses based on `ETag` response header -* Set `Date` header for any received response missing it -* Support `max-stale` cache-control directive in requests +* Do not attempt to store responses/requests with ``no-store`` cache-control directive set +* Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses +* Compute freshness lifetime from ``max-age`` cache-control directive +* Compute freshness lifetime from ``Expires`` response header +* Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox) +* Compute current age from ``Age`` response header +* Compute current age from ``Date`` header +* Revalidate stale responses based on ``Last-Modified`` response header +* Revalidate stale responses based on ``ETag`` response header +* Set ``Date`` header for any received response missing it +* Support ``max-stale`` cache-control directive in requests This allows spiders to be configured with the full RFC2616 cache policy, but avoid revalidation on a request-by-request basis, while remaining @@ -408,15 +408,15 @@ what is implemented: Example: - Add `Cache-Control: max-stale=600` to Request headers to accept responses that + Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that have exceeded their expiration time by no more than 600 seconds. See also: RFC2616, 14.9.3 what is missing: -* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 -* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 +* ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 +* ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 * ... probably others .. @@ -626,12 +626,12 @@ Default: ``False`` If enabled, will cache pages unconditionally. A spider may wish to have all responses available in the cache, for -future use with `Cache-Control: max-stale`, for instance. The +future use with ``Cache-Control: max-stale``, for instance. The DummyPolicy caches all responses but never revalidates them, and sometimes a more nuanced policy is desirable. -This setting still respects `Cache-Control: no-store` directives in responses. -If you don't want that, filter `no-store` out of the Cache-Control headers in +This setting still respects ``Cache-Control: no-store`` directives in responses. +If you don't want that, filter ``no-store`` out of the Cache-Control headers in responses you feedto the cache middleware. .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS @@ -940,7 +940,7 @@ UserAgentMiddleware Middleware that allows spiders to override the default user agent. - In order for a spider to override the default user agent, its `user_agent` + In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. .. _ajaxcrawl-middleware: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 95f7920f8..f5048d2da 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -303,7 +303,7 @@ CsvItemExporter The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor, and the leftover arguments to the - `csv.writer`_ constructor, so you can use any `csv.writer` constructor + `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor argument to customize this exporter. A typical output of this exporter would be:: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index c421a5e05..d6e7452a1 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -19,7 +19,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a hypothetic extension to handle `Google Sitemaps`_ would use settings like -`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on. +``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps @@ -368,7 +368,7 @@ Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues running normally. -For more info see `Debugging in Python`. +For more info see `Debugging in Python`_. This extension only works on POSIX-compliant platforms (ie. not Windows). diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index ea684b4cf..1a5d52487 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -71,7 +71,7 @@ on cookies. Request serialization --------------------- -Requests must be serializable by the `pickle` module, in order for persistence +Requests must be serializable by the ``pickle`` module, in order for persistence to work, so you should make sure that your requests are serializable. The most common issue here is to use ``lambda`` functions on request callbacks that diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index f3b6aa4a1..1c2f1da4d 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -286,7 +286,7 @@ ItemLoader objects given, one is instantiated automatically using the class in :attr:`default_item_class`. - When instantiated with a `selector` or a `response` parameters + When instantiated with a ``selector`` or a ``response`` parameters the :class:`ItemLoader` class provides convenient mechanisms for extracting data from web pages using :ref:`selectors `. diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 0986929ad..8e280d929 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -243,7 +243,7 @@ scrapy.utils.log module case, its usage is not required but it's recommended. If you plan on configuring the handlers yourself is still recommended you - call this function, passing `install_root_handler=False`. Bear in mind + call this function, passing ``install_root_handler=False``. Bear in mind there won't be any log output set by default in that case. To get you started on manually configuring logging's output, you can use diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index c60b55391..381a2988a 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -132,7 +132,7 @@ For example, the following image URL:: http://www.example.com/image.jpg -Whose `SHA1 hash` is:: +Whose ``SHA1 hash`` is:: 3afec3b4765f8f0a07b78f98c07b83f013567a0a diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 02cfa9b05..298a078a7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -80,7 +80,7 @@ returned by the :meth:`CrawlerRunner.crawl ` method. Here's an example of its usage, along with a callback to manually stop the -reactor after `MySpider` has finished running. +reactor after ``MySpider`` has finished running. :: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 76360b15f..8b3ba4f2d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -50,7 +50,7 @@ Request objects :type meta: dict :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the `encoding` passed (which defaults to ``utf-8``). If + ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). @@ -610,7 +610,7 @@ Response objects .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for - tagging Responses. For example: `'cached'`, `'redirected`', etc. And + tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. @@ -682,7 +682,7 @@ TextResponse objects ``unicode(response.body)`` is not a correct way to convert response body to unicode: you would be using the system default encoding - (typically `ascii`) instead of the response encoding. + (typically ``ascii``) instead of the response encoding. .. attribute:: TextResponse.encoding @@ -690,7 +690,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the constructor `encoding` argument + 1. the encoding passed in the constructor ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (ie. unknown), it is ignored and the next diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index df1d67ae8..edc18f14d 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -96,7 +96,7 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of Using selectors --------------- -To explain how to use the selectors we'll use the `Scrapy shell` (which +To explain how to use the selectors we'll use the ``Scrapy shell`` (which provides interactive testing) and an example page located in the Scrapy documentation server: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..1afa513c8 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -599,7 +599,7 @@ The amount of time (in secs) that the downloader will wait before timing out. DOWNLOAD_MAXSIZE ---------------- -Default: `1073741824` (1024MB) +Default: ``1073741824`` (1024MB) The maximum response size (in bytes) that downloader will download. @@ -620,7 +620,7 @@ If you want to disable it set to 0. DOWNLOAD_WARNSIZE ----------------- -Default: `33554432` (32MB) +Default: ``33554432`` (32MB) The response size (in bytes) that downloader will start to warn. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..b551aa47d 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied. If you want to disable a builtin middleware (the ones defined in :setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it -in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its +in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the off-site middleware:: SPIDER_MIDDLEWARES = { @@ -200,7 +200,7 @@ DepthMiddleware .. class:: DepthMiddleware DepthMiddleware is used for tracking the depth of each Request inside the - site being scraped. It works by setting `request.meta['depth'] = 0` whenever + site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever there is no value previously set (usually just the first Request) and incrementing it by 1 otherwise. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..09feedefc 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -129,7 +129,7 @@ scrapy.Spider You probably won't need to override this directly because the default implementation acts as a proxy to the :meth:`__init__` method, calling - it with the given arguments `args` and named arguments `kwargs`. + it with the given arguments ``args`` and named arguments ``kwargs``. Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` attributes in the new instance so they can be accessed later inside the @@ -298,13 +298,13 @@ The above example can also be written as follows:: Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. -If you were to set the `start_urls` attribute from the command line, +If you were to set the ``start_urls`` attribute from the command line, you would have to parse it on your own into a list using something like `ast.literal_eval `_ or `json.loads `_ and then set it as an attribute. -Otherwise, you would cause iteration over a `start_urls` string +Otherwise, you would cause iteration over a ``start_urls`` string (a very common python pitfall) resulting in each character being seen as a separate url. diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 81ce800aa..6c993a970 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -22,7 +22,7 @@ To use the packages: sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7 -2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command:: +2. Create ``/etc/apt/sources.list.d/scrapy.list`` file using the following command:: echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list @@ -34,7 +34,7 @@ To use the packages: .. note:: Repeat step 3 if you are trying to upgrade Scrapy. -.. warning:: `python-scrapy` is a different package provided by official debian +.. warning:: ``python-scrapy`` is a different package provided by official debian repositories, it's very outdated and it isn't supported by Scrapy team. .. _Scrapinghub: https://scrapinghub.com/ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04aee18ed..2ecc4daad 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -153,7 +153,7 @@ class CrawlerRunner(object): It will call the given Crawler's :meth:`~Crawler.crawl` method, while keeping track of it so it can be stopped later. - If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler` + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as the spider class given to it. @@ -188,10 +188,10 @@ class CrawlerRunner(object): """ Return a :class:`~scrapy.crawler.Crawler` object. - * If `crawler_or_spidercls` is a Crawler, it is returned as-is. - * If `crawler_or_spidercls` is a Spider subclass, a new Crawler + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler is constructed for it. - * If `crawler_or_spidercls` is a string, this function finds + * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. """ @@ -273,7 +273,7 @@ class CrawlerProcess(CrawlerRunner): :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. - If `stop_after_crawl` is True, the reactor will be stopped after all + If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 075a6d862..65f347dcf 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -13,21 +13,21 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - All methods must return a dictionary listing the parameters `level`, `msg` - and `args` which are going to be used for constructing the log message when - calling logging.log. + All methods must return a dictionary listing the parameters ``level``, + ``msg`` and ``args`` which are going to be used for constructing the log + message when calling logging.log. Dictionary keys for the method outputs: - * `level` should be the log level for that action, you can use those + * ``level`` should be the log level for that action, you can use those from the python logging library: logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR and logging.CRITICAL. - * `msg` should be a string that can contain different formatting - placeholders. This string, formatted with the provided `args`, is going - to be the log message for that action. + * ``msg`` should be a string that can contain different formatting + placeholders. This string, formatted with the provided ``args``, is + going to be the log message for that action. - * `args` should be a tuple or dict with the formatting placeholders for - `msg`. The final log message is computed as output['msg'] % + * ``args`` should be a tuple or dict with the formatting placeholders + for ``msg``. The final log message is computed as output['msg'] % output['args']. """ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 510cc23c7..2d8091f5b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -255,13 +255,13 @@ class FilesPipeline(MediaPipeline): doing stat of the files and determining if file is new, uptodate or expired. - `new` files are those that pipeline never processed and needs to be + ``new`` files are those that pipeline never processed and needs to be downloaded from supplier site the first time. - `uptodate` files are the ones that the pipeline processed and are still + ``uptodate`` files are the ones that the pipeline processed and are still valid files. - `expired` files are those that pipeline already processed but the last + ``expired`` files are those that pipeline already processed but the last modification was made long time ago, so a reprocessing is recommended to refresh it in case of change. diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f255d436f..9eca6a4da 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -2,7 +2,7 @@ from ftplib import error_perm from posixpath import dirname def ftp_makedirs_cwd(ftp, path, first_call=True): - """Set the current directory of the FTP connection given in the `ftp` + """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. """ diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 828880709..e07fb8698 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -32,7 +32,7 @@ class TopLevelFormatter(logging.Filter): Since it can't be set for just one logger (it won't propagate for its children), it's going to be set in the root handler, with a parametrized - `loggers` list where it should act. + ``loggers`` list where it should act. """ def __init__(self, loggers=None): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 732ca13a0..aade3d9ac 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -97,8 +97,8 @@ def unicode_to_str(text, encoding=None, errors='strict'): def to_unicode(text, encoding=None, errors='strict'): - """Return the unicode representation of a bytes object `text`. If `text` - is already an unicode object, return it as-is.""" + """Return the unicode representation of a bytes object ``text``. If + ``text`` is already an unicode object, return it as-is.""" if isinstance(text, six.text_type): return text if not isinstance(text, (bytes, six.text_type)): @@ -110,7 +110,7 @@ def to_unicode(text, encoding=None, errors='strict'): def to_bytes(text, encoding=None, errors='strict'): - """Return the binary representation of `text`. If `text` + """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text @@ -123,7 +123,7 @@ def to_bytes(text, encoding=None, errors='strict'): def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of `text` + """ Return str representation of ``text`` (bytes in Python 2.x and unicode in Python 3.x). """ if six.PY2: return to_bytes(text, encoding, errors) @@ -189,7 +189,7 @@ def isbinarytext(text): def binary_is_text(data): - """ Returns `True` if the given ``data`` argument (a ``bytes`` object) + """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ if not isinstance(data, bytes): @@ -314,7 +314,7 @@ class WeakKeyCache(object): @deprecated def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """Return a (new) dict with unicode keys (and values when "keys_only" is - False) of the given dict converted to strings. `dct_or_tuples` can be a + False) of the given dict converted to strings. ``dct_or_tuples`` can be a dict or a list of tuples, like any dict constructor supports. """ d = {} @@ -357,10 +357,10 @@ def retry_on_eintr(function, *args, **kw): def without_none_values(iterable): - """Return a copy of `iterable` with all `None` entries removed. + """Return a copy of ``iterable`` with all ``None`` entries removed. - If `iterable` is a mapping, return a dictionary where all pairs that have - value `None` have been removed. + If ``iterable`` is a mapping, return a dictionary where all pairs that have + value ``None`` have been removed. """ try: return {k: v for k, v in six.iteritems(iterable) if v is not None} diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 657c53815..b3a4be007 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -109,12 +109,12 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= """Strip URL string from some of its components: - - `strip_credentials` removes "user:password@" - - `strip_default_port` removes ":80" (resp. ":443", ":21") + - ``strip_credentials`` removes "user:password@" + - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - - `origin_only` replaces path component with "/", also dropping + - ``origin_only`` replaces path component with "/", also dropping query and fragment components ; it also strips credentials - - `strip_fragment` drops any #fragment component + - ``strip_fragment`` drops any #fragment component """ parsed_url = urlparse(url) diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 366fcf033..eb362e945 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -10,7 +10,8 @@ Status Obsolete (discarded) SEP-006: Rename of Selectors to Extractors ========================================== -This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and their `x` method. +This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and +their ``x`` method. Motivation ========== @@ -57,7 +58,7 @@ Additional changes As the name of the method for performing selection (the ``x`` method) is not descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method (x sounds like a short for extract in english), we propose to rename it to -`select`, `sel` (is shortness if required), or `xpath` after `lxml's +``select``, ``sel`` (is shortness if required), or ``xpath`` after `lxml's `_ ``xpath`` method. Bonus (ItemBuilder) diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 40d9293b2..431428331 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -16,7 +16,7 @@ _DATABASES = collections.defaultdict(DummyDB) def open(file, flag='r', mode=0o666): """Open or create a dummy database compatible. - Arguments `flag` and `mode` are ignored. + Arguments ``flag`` and ``mode`` are ignored. """ # return same instance for same file argument return _DATABASES[file] diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 36baacfbd..d664b6ade 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -61,7 +61,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_fetch_redirect_follow_302(self): - """Test that calling `fetch(url)` follows HTTP redirects by default.""" + """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') code = "fetch('{0}')" errcode, out, errout = yield self.execute(['-c', code.format(url)]) @@ -71,7 +71,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_fetch_redirect_not_follow_302(self): - """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') code = "fetch('{0}', redirect=False)" errcode, out, errout = yield self.execute(['-c', code.format(url)]) From 6eca6f92c6ae48bb136311308e77e82d7659cf43 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 14:59:34 +0530 Subject: [PATCH 0311/1853] Update form.py --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index af4ed1793..b6ca6ef9c 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,11 +34,8 @@ class FormRequest(Request): self._set_body(querystr) else: url_split = urlsplit(self.url) - formdata_key_list = [] - for k in querystr.split('&'): - formdata_key_list.append(k.split('=')[0]) - items = [] - items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + formdata_key_list = list(dict(parse_qsl(querystr)).keys()) + items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] query_str = _urlencode(items, self.encoding) self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) From d75b61b96aad172e21c7d3aeba6f48419a63c72d Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:07:12 +0530 Subject: [PATCH 0312/1853] Update test_http_request.py --- tests/test_http_request.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 955db9e76..2da1cdf6d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -289,6 +289,12 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) + #GET duplicates are preserved + formdata=(('foo', 'bar'), ('foo', 'baz')) + fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) + self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'}) + self.assertEqual(set(fs[b'a']), {b'1', b'2'}) + def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From fdf03a6d0daae79b9a3b421d19102873d5a75f46 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:12:44 +0530 Subject: [PATCH 0313/1853] correcting tests --- tests/test_http_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 2da1cdf6d..0c2f95262 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -289,10 +289,10 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #GET duplicates are preserved - formdata=(('foo', 'bar'), ('foo', 'baz')) + #Duplicate GET arguments are preserved + formdata={'foo' : 'bar'} fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) - self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'}) + self.assertEqual(fs[b'foo'], [b'bar']) self.assertEqual(set(fs[b'a']), {b'1', b'2'}) def test_default_encoding_bytes(self): From 8831fafabc699bd2ab48e3e185712fbcd4f7cc7f Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:42:48 +0530 Subject: [PATCH 0314/1853] Update test_http_request.py --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0c2f95262..5c30c9785 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -290,7 +290,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'b'], [b'2']) #Duplicate GET arguments are preserved - formdata={'foo' : 'bar'} + data={'foo' : 'bar'} fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) self.assertEqual(fs[b'foo'], [b'bar']) self.assertEqual(set(fs[b'a']), {b'1', b'2'}) From 7da460b7935940c93d5454019ea99f9f117b3e7b Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 17:25:15 +0530 Subject: [PATCH 0315/1853] Update form.py --- scrapy/http/request/form.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6ca6ef9c..b44d7b9a6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit import lxml.html from parsel.selector import create_root_node @@ -34,10 +34,10 @@ class FormRequest(Request): self._set_body(querystr) else: url_split = urlsplit(self.url) - formdata_key_list = list(dict(parse_qsl(querystr)).keys()) - items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + formdata_keys = set(dict(parse_qsl(querystr)).keys()) + items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] query_str = _urlencode(items, self.encoding) - self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) + self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 82d239f3b148d9ce69f67bd7a2cb00de7e934aa6 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Wed, 6 Mar 2019 12:08:09 +0530 Subject: [PATCH 0316/1853] docs for scrapy.logformatter --- docs/topics/logging.rst | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 0986929ad..a5fecebba 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -193,6 +193,45 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +Custom Log Formats +------------------- + +Custom log format can be set for different actions by extending ``scrapy.logformatter.LogFormatter`` class. + +Each method of ``scrapy.logformatter.LogFormatter`` represents an action. All methods inherited from +``scrapy.logformatter.LogFormatter`` in your custom log formatting class must return a dictionary listing +the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message. +Listed below is details of what each key represents : + +* ``level`` is the log level for that action, you can use those from the python logging library: + :setting:`logging.DEBUG`, :setting:`logging.INFO`, :setting:`logging.WARNING`, :setting:`logging.ERROR` + and :setting:`logging.CRITICAL`. + +* ``msg`` should be a string that can contain different formatting placeholders. This string, formatted + with the provided ``args``, is going to be the long message for that action. + +* ``args`` should be a tuple or dict with the formatting placeholders for `msg`. The final log message is + computed as ``msg % args``. + +.. note:: To use custom log formatting class, you must mention it in ``settings.py``, by adding a line + ``LOG_FORMATTER = '’`` + +.. class:: scrapy.logformatter.LogFormatter + + The default log formatting class in Scrapy. + + .. method:: crawled (request, response, spider) + + ``crawled`` is called to log message when the crawler finds a webpage. + + .. method:: scraped(item, response, spider) + + ``scraped`` is called to log message when an item scraped by a spider. + + .. method:: dropped(item, exception, response, spider) + + ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. + Advanced customization ---------------------- From f7bf3abbd03a10124ed648d45abbb6c8eb3218b7 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 6 Mar 2019 14:10:03 +0530 Subject: [PATCH 0317/1853] Modified code --- scrapy/http/request/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b44d7b9a6..a0e7f76a9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -37,7 +37,8 @@ class FormRequest(Request): formdata_keys = set(dict(parse_qsl(querystr)).keys()) items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] query_str = _urlencode(items, self.encoding) - self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment))) + query = (query_str + '&' + querystr) if query_str else querystr + self._set_url(urlunsplit(url_split._replace(query = query))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 184def1060f95767ef767948edeb3bbb4ed0e428 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 7 Mar 2019 00:09:10 +0530 Subject: [PATCH 0318/1853] fix a link inside docs --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4ac39ad2d..2effe94dc 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -172,5 +172,5 @@ links: .. _Twisted: https://twistedmatrix.com/trac/ .. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/ +.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ From 924b67437b92f14601816d02c5d153e7281da6d4 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 7 Mar 2019 16:40:59 +0530 Subject: [PATCH 0319/1853] move api docs to source code --- docs/topics/logging.rst | 38 ++++---------------------------------- docs/topics/settings.rst | 9 +++++++++ scrapy/logformatter.py | 36 +++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 47 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a5fecebba..72f24bae6 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -196,41 +196,11 @@ to override some of the Scrapy settings regarding logging. Custom Log Formats ------------------- -Custom log format can be set for different actions by extending ``scrapy.logformatter.LogFormatter`` class. - -Each method of ``scrapy.logformatter.LogFormatter`` represents an action. All methods inherited from -``scrapy.logformatter.LogFormatter`` in your custom log formatting class must return a dictionary listing -the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message. -Listed below is details of what each key represents : - -* ``level`` is the log level for that action, you can use those from the python logging library: - :setting:`logging.DEBUG`, :setting:`logging.INFO`, :setting:`logging.WARNING`, :setting:`logging.ERROR` - and :setting:`logging.CRITICAL`. - -* ``msg`` should be a string that can contain different formatting placeholders. This string, formatted - with the provided ``args``, is going to be the long message for that action. - -* ``args`` should be a tuple or dict with the formatting placeholders for `msg`. The final log message is - computed as ``msg % args``. - -.. note:: To use custom log formatting class, you must mention it in ``settings.py``, by adding a line - ``LOG_FORMATTER = '’`` +Custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class +and making :setting:`LOG_FORMATTER` inside ``settings.py`` point to your new class. -.. class:: scrapy.logformatter.LogFormatter - - The default log formatting class in Scrapy. - - .. method:: crawled (request, response, spider) - - ``crawled`` is called to log message when the crawler finds a webpage. - - .. method:: scraped(item, response, spider) - - ``scraped`` is called to log message when an item scraped by a spider. - - .. method:: dropped(item, exception, response, spider) - - ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. +.. autoclass:: scrapy.logformatter.LogFormatter + :members: Advanced customization ---------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..1dfb5b8aa 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -866,6 +866,15 @@ directives. .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. setting:: LOG_FORMATTER + +LOG_FORMATTER +------------- + +Default: ``scrapy.logformatter.LogFormatter`` + +The class to use for formatting log messages for different actions. + .. setting:: LOG_LEVEL LOG_LEVEL diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 075a6d862..0bb8aee58 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -13,25 +13,29 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - All methods must return a dictionary listing the parameters `level`, `msg` - and `args` which are going to be used for constructing the log message when - calling logging.log. + All methods must return a dictionary listing the parameters ``level``, ``msg`` + and ``args`` which are going to be used for constructing the log message when + calling ``logging.log``. Dictionary keys for the method outputs: - * `level` should be the log level for that action, you can use those - from the python logging library: logging.DEBUG, logging.INFO, - logging.WARNING, logging.ERROR and logging.CRITICAL. - * `msg` should be a string that can contain different formatting - placeholders. This string, formatted with the provided `args`, is going - to be the log message for that action. + * ``level`` is the log level for that action, you can use those from the + `python logging library `_ : + ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` + and ``logging.CRITICAL``. + + * ``msg`` should be a string that can contain different formatting placeholders. This string, formatted + with the provided ``args``, is going to be the long message for that action. + + * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is + computed as ``msg % args``. - * `args` should be a tuple or dict with the formatting placeholders for - `msg`. The final log message is computed as output['msg'] % - output['args']. """ def crawled(self, request, response, spider): + """ + ``crawled`` is called to log message when the crawler finds a webpage. + """ request_flags = ' %s' % str(request.flags) if request.flags else '' response_flags = ' %s' % str(response.flags) if response.flags else '' return { @@ -40,7 +44,7 @@ class LogFormatter(object): 'args': { 'status': response.status, 'request': request, - 'request_flags' : request_flags, + 'request_flags': request_flags, 'referer': referer_str(request), 'response_flags': response_flags, # backward compatibility with Scrapy logformatter below 1.4 version @@ -49,6 +53,9 @@ class LogFormatter(object): } def scraped(self, item, response, spider): + """ + ``scraped`` is called to log message when an item is scraped by a spider. + """ if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -63,6 +70,9 @@ class LogFormatter(object): } def dropped(self, item, exception, response, spider): + """ + ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. + """ return { 'level': logging.WARNING, 'msg': DROPPEDMSG, From 120007c0577f819a26ff795ac8b9cd6fc397df19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 13:53:47 +0100 Subject: [PATCH 0320/1853] Add a FAQ entry on how to deal with long lists of allowed domains --- docs/faq.rst | 35 +++++++++++++++++++++++++++++++ docs/topics/spider-middleware.rst | 2 ++ 2 files changed, 37 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..f56d26c0a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,6 +149,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..80357a987 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -54,6 +54,8 @@ value. For example, if you want to disable the off-site middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _custom-spider-middleware: + Writing your own spider middleware ================================== From 4ef38d925e0ded380a7cebabe3aab2340d4f3d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 14:21:00 +0100 Subject: [PATCH 0321/1853] Remove the unexisting retry_complete signal from the documentation --- docs/topics/downloader-middleware.rst | 2 -- scrapy/downloadermiddlewares/retry.py | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8dbe249fa..9988ab18b 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -834,8 +834,6 @@ RetryMiddleware Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. -Once there are no more failed pages to retry, this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. The :class:`RetryMiddleware` can be configured through the following settings (see the settings documentation for more info): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 07e979628..dbc605a4c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,9 +7,7 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. Once -there is no more failed pages to retry this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. +once the spider has finished crawling all regular (non failed) pages. """ import logging From e108e3adbfa1a1f9bdd2a84180f18e5e43a39d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 15:13:11 +0100 Subject: [PATCH 0322/1853] Clarify the documentation of DEPTH_PRIORITY further --- docs/topics/settings.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..229a9e956 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -331,16 +331,16 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the request priority based on its depth: +An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of +a :class:`~scrapy.http.Request` based on its depth. -- if zero (default), no priority adjustment is made from depth -- **a positive value will decrease the priority, i.e. higher depth - requests will be processed later** ; this is commonly used when doing - breadth-first crawls (BFO) -- a negative value will increase priority, i.e., higher depth requests - will be processed sooner (DFO) +The priority of a request is adjusted as follows:: -See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. + request.priority = request.priority - ( depth * DEPTH_PRIORITY ) + +As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request +priority (BFO), while negative values increase request priority (DFO). See +also :ref:`faq-bfo-dfo`. .. note:: From b1063d9b2ca1a6bfb947fdd8b0158633184114ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 17:22:49 +0100 Subject: [PATCH 0323/1853] Use the description from README.rst on index.rst --- docs/index.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 0a96aa88e..cedde8f38 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,13 @@ Scrapy |version| documentation ============================== -This documentation contains everything you need to know about Scrapy. +Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used +to crawl websites and extract structured data from their pages. It can be used +for a wide range of purposes, from data mining to monitoring and automated +testing. + +.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler +.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping Getting help ============ From 91aec8b3bb805e9595f1b778fb14f703c6acf2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 18:19:30 +0100 Subject: [PATCH 0324/1853] Update developer-tools.rst Fixes #3674 --- docs/topics/developer-tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c1976258d..82857c9da 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -233,7 +233,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1] + start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) From 82049e9c41f878d84f0fe10f827c6fe2a33f7ba6 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sun, 10 Mar 2019 20:14:55 +0530 Subject: [PATCH 0325/1853] make suggested changes. --- docs/topics/logging.rst | 10 ++++++---- docs/topics/settings.rst | 4 ++-- scrapy/logformatter.py | 26 +++++++++++++++++--------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 72f24bae6..006530a8c 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -193,11 +193,13 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers -Custom Log Formats -------------------- +.. _custom-log-formats: -Custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class -and making :setting:`LOG_FORMATTER` inside ``settings.py`` point to your new class. +Custom Log Formats +------------------ + +A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class +and making :setting:`LOG_FORMATTER` point to your new class. .. autoclass:: scrapy.logformatter.LogFormatter :members: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1dfb5b8aa..a36c0b34c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -871,9 +871,9 @@ directives. LOG_FORMATTER ------------- -Default: ``scrapy.logformatter.LogFormatter`` +Default: :class:`scrapy.logformatter.LogFormatter` -The class to use for formatting log messages for different actions. +The class to use for :ref:`formatting log messages ` for different actions. .. setting:: LOG_LEVEL diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 0bb8aee58..17c69cba8 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -30,12 +30,24 @@ class LogFormatter(object): * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is computed as ``msg % args``. + Here is an example on how to create a custom log formatter to lower the severity level of the log message + when an item is dropped from the pipeline:: + + class PoliteLogFormatter(logformatter.LogFormatter): + def dropped(self, item, exception, response, spider): + return { + 'level': logging.INFO, # lowering the level from logging.WARNING + 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'args': { + 'exception': exception, + 'item': item, + } + } + """ def crawled(self, request, response, spider): - """ - ``crawled`` is called to log message when the crawler finds a webpage. - """ + """Logs a message when the crawler finds a webpage.""" request_flags = ' %s' % str(request.flags) if request.flags else '' response_flags = ' %s' % str(response.flags) if response.flags else '' return { @@ -53,9 +65,7 @@ class LogFormatter(object): } def scraped(self, item, response, spider): - """ - ``scraped`` is called to log message when an item is scraped by a spider. - """ + """Logs a message when an item is scraped by a spider.""" if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -70,9 +80,7 @@ class LogFormatter(object): } def dropped(self, item, exception, response, spider): - """ - ``dropped`` is called to log message when an item is dropped while it is passing through the item pipeline. - """ + """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { 'level': logging.WARNING, 'msg': DROPPEDMSG, From e9cd4ee03aa41e27bea0408b10970ec5bedf35d3 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sun, 10 Mar 2019 20:37:56 +0530 Subject: [PATCH 0326/1853] fix list alignment and line width --- scrapy/logformatter.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 17c69cba8..717120242 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -19,19 +19,18 @@ class LogFormatter(object): Dictionary keys for the method outputs: - * ``level`` is the log level for that action, you can use those from the - `python logging library `_ : - ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` - and ``logging.CRITICAL``. + * ``level`` is the log level for that action, you can use those from the + `python logging library `_ : + ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` + and ``logging.CRITICAL``. + * ``msg`` should be a string that can contain different formatting placeholders. + This string, formatted with the provided ``args``, is going to be the long message + for that action. + * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. + The final log message is computed as ``msg % args``. - * ``msg`` should be a string that can contain different formatting placeholders. This string, formatted - with the provided ``args``, is going to be the long message for that action. - - * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. The final log message is - computed as ``msg % args``. - - Here is an example on how to create a custom log formatter to lower the severity level of the log message - when an item is dropped from the pipeline:: + Here is an example on how to create a custom log formatter to lower the severity level of + the log message when an item is dropped from the pipeline:: class PoliteLogFormatter(logformatter.LogFormatter): def dropped(self, item, exception, response, spider): From 35f7595dbebe7012c8f66336b76625b089a1fad3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 11 Mar 2019 23:58:37 +0530 Subject: [PATCH 0327/1853] changed variable names --- scrapy/http/request/form.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a0e7f76a9..9af2db5ff 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -28,16 +28,16 @@ class FormRequest(Request): if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata - querystr = _urlencode(items, self.encoding) + form_query_str = _urlencode(items, self.encoding) if self.method == 'POST': self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') - self._set_body(querystr) + self._set_body(form_query_str) else: url_split = urlsplit(self.url) - formdata_keys = set(dict(parse_qsl(querystr)).keys()) + formdata_keys = set(dict(parse_qsl(form_query_str)).keys()) items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] - query_str = _urlencode(items, self.encoding) - query = (query_str + '&' + querystr) if query_str else querystr + url_query_str = _urlencode(items, self.encoding) + query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str self._set_url(urlunsplit(url_split._replace(query = query))) @classmethod From 43fd6229684b3ccca564524fc92faf009a8c4c97 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 10:21:50 +0000 Subject: [PATCH 0328/1853] Rule.process_request: optionally take a Response object --- scrapy/spiders/crawl.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index e5ac72e18..5aec0fd83 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -24,12 +24,23 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request = process_request + self.process_request_function = process_request if follow is None: self.follow = False if callback else True else: self.follow = follow + def process_request(self, request, response): + """ + Wrapper around the request processing function to maintain backward compatibility + with functions that do not take a Response object as parameter. + """ + argcount = self.process_request_function.__code__.co_argcount + if getattr(self.process_request_function, '__self__', None): + argcount = argcount - 1 + args = [request] if argcount == 1 else [request, response] + return self.process_request_function(*args) + class CrawlSpider(Spider): @@ -65,7 +76,7 @@ class CrawlSpider(Spider): for link in links: seen.add(link) r = self._build_request(n, link) - yield rule.process_request(r) + yield rule.process_request(r, response) def _response_downloaded(self, response): rule = self._rules[response.meta['rule']] @@ -93,7 +104,7 @@ class CrawlSpider(Spider): for rule in self._rules: rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) - rule.process_request = get_method(rule.process_request) + rule.process_request_function = get_method(rule.process_request_function) @classmethod def from_crawler(cls, crawler, *args, **kwargs): From 22fda61d62a2b230b0e8588eabb0d71cb77141b7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 10:54:38 +0000 Subject: [PATCH 0329/1853] Rule.process_request: tests --- tests/test_spider.py | 98 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index fefdaa403..5e20e0d99 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -263,6 +263,104 @@ class CrawlSpiderTest(SpiderTest): 'http://example.org/about.html', 'http://example.org/nofollow.html']) + def test_process_request(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + def process_request_change_domain(request): + return request.replace(url=request.url.replace('.org', '.com')) + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request=process_request_change_domain), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) + + def test_process_request_with_response(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + def process_request_meta_response_class(request, response): + request.meta['response_class'] = response.__class__.__name__ + return request + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request=process_request_meta_response_class), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) + self.assertEqual([r.meta['response_class'] for r in output], + ['HtmlResponse', 'HtmlResponse', 'HtmlResponse']) + + def test_process_request_instance_method(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request='process_request_upper'), + ) + + def process_request_upper(self, request): + return request.replace(url=request.url.upper()) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + + def test_process_request_instance_method_with_response(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request='process_request_meta_response_class'), + ) + + def process_request_meta_response_class(self, request, response): + request.meta['response_class'] = response.__class__.__name__ + return request + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) + self.assertEqual([r.meta['response_class'] for r in output], + ['HtmlResponse', 'HtmlResponse', 'HtmlResponse']) + def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, 'example.com') From b30ca379b6785c7ceb75e12285fe7865b4f607d1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 11:02:51 +0000 Subject: [PATCH 0330/1853] Rule.process_request: docs --- docs/topics/spiders.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..24b6f7ec9 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -402,10 +402,12 @@ Crawling rules of links extracted from each response using the specified ``link_extractor``. This is mainly used for filtering purposes. - ``process_request`` is a callable, or a string (in which case a method from - the spider object with that name will be used) which will be called with - every request extracted by this rule, and must return a request or None (to - filter out the request). + ``process_request`` is a callable (or a string, in which case a method from + the spider object with that name will be used) which will be called for + every request extracted by this rule. This callable should take a Request object + as first positional argument and, optionally, the Response object from which the + Request originated as second positional argument. It must return a request or None + (to filter out the request). CrawlSpider example ~~~~~~~~~~~~~~~~~~~ From 83ec947fe732035e147c21df352e199ce2cce5c8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 11:23:51 +0000 Subject: [PATCH 0331/1853] Rule.process_request defaults to None in the docs --- scrapy/spiders/crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 5aec0fd83..ad86fc19d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -19,12 +19,12 @@ def identity(x): class Rule(object): - def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity): + def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): self.link_extractor = link_extractor self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request_function = process_request + self.process_request_function = process_request or identity if follow is None: self.follow = False if callback else True else: From a2ff647aace899982bd494d73fea5a7c35ded722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Mar 2019 06:36:15 +0100 Subject: [PATCH 0332/1853] Remove docs/topics/ubuntu.rst --- docs/topics/ubuntu.rst | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 docs/topics/ubuntu.rst diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst deleted file mode 100644 index 6c993a970..000000000 --- a/docs/topics/ubuntu.rst +++ /dev/null @@ -1,41 +0,0 @@ -:orphan: Ubuntu packages are obsolete - -.. _topics-ubuntu: - -=============== -Ubuntu packages -=============== - -.. versionadded:: 0.10 - -`Scrapinghub`_ publishes apt-gettable packages which are generally fresher than -those in Ubuntu, and more stable too since they're continuously built from -`GitHub repo`_ (master & stable branches) and so they contain the latest bug -fixes. - -.. caution:: These packages are currently not updated and may not work on - Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`. - -To use the packages: - -1. Import the GPG key used to sign Scrapy packages into APT keyring:: - - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7 - -2. Create ``/etc/apt/sources.list.d/scrapy.list`` file using the following command:: - - echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list - -3. Update package lists and install the scrapy package: - - .. parsed-literal:: - - sudo apt-get update && sudo apt-get install scrapy - -.. note:: Repeat step 3 if you are trying to upgrade Scrapy. - -.. warning:: ``python-scrapy`` is a different package provided by official debian - repositories, it's very outdated and it isn't supported by Scrapy team. - -.. _Scrapinghub: https://scrapinghub.com/ -.. _GitHub repo: https://github.com/scrapy/scrapy From 01ed605d02013b1d7955369562b2443d2a561599 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 Mar 2019 16:54:14 +0000 Subject: [PATCH 0333/1853] PEP8 changes to test_spider.py --- tests/test_spider.py | 60 +++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 5e20e0d99..c9af7a2d7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -105,11 +105,11 @@ class SpiderTest(unittest.TestCase): def test_logger(self): spider = self.spider_class('example.com') - with LogCapture() as l: + with LogCapture() as lc: spider.logger.info('test log msg') - l.check(('example.com', 'INFO', 'test log msg')) + lc.check(('example.com', 'INFO', 'test log msg')) - record = l.records[0] + record = lc.records[0] self.assertIn('spider', record.__dict__) self.assertIs(record.spider, spider) @@ -190,12 +190,11 @@ class CrawlSpiderTest(SpiderTest): def test_process_links(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="dummy_process_links"), ) @@ -208,24 +207,24 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) def test_process_links_filter(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): import re - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="filter_process_links"), ) _test_regex = re.compile('nofollow') + def filter_process_links(self, links): return [link for link in links if not self._test_regex.search(link.url)] @@ -235,17 +234,16 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html']) def test_process_links_generator(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="dummy_process_links"), ) @@ -259,9 +257,9 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) def test_process_request(self): @@ -271,8 +269,8 @@ class CrawlSpiderTest(SpiderTest): return request.replace(url=request.url.replace('.org', '.com')) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request=process_request_change_domain), ) @@ -295,8 +293,8 @@ class CrawlSpiderTest(SpiderTest): return request class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request=process_request_meta_response_class), ) @@ -317,8 +315,8 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request='process_request_upper'), ) @@ -340,8 +338,8 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request='process_request_meta_response_class'), ) From 645e8d16a4c966b50bd39667aaef28dc1eeb43b8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 Mar 2019 22:20:36 +0000 Subject: [PATCH 0334/1853] Count keyword argument checks --- tests/spiders.py | 21 +++++++++++++-------- tests/test_crawl.py | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 5a1471072..7b4707f62 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -31,7 +31,7 @@ class MetaSpider(MockServerSpider): class KeywordArgumentsSpider(MockServerSpider): name = 'kwargs' - checks = set() + checks = list() def start_requests(self): data = {'key': 'value', 'number': 123} @@ -41,25 +41,30 @@ class KeywordArgumentsSpider(MockServerSpider): yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) def parse_first(self, response, key, number): - self.checks.add(key == 'value') - self.checks.add(number == 123) + self.checks.append(key == 'value') + self.checks.append(number == 123) + self.crawler.stats.inc_value('boolean_checks', 2) yield response.follow( self.mockserver.url('/two'), self.parse_second, kwargs={'new_key': 'new_value'}) def parse_second(self, response, new_key): - self.checks.add(new_key == 'new_value') + self.checks.append(new_key == 'new_value') + self.crawler.stats.inc_value('boolean_checks') def parse_general(self, response, **kwargs): if response.url.endswith('/general_with'): - self.checks.add(kwargs['key'] == 'value') - self.checks.add(kwargs['number'] == 123) + self.checks.append(kwargs['key'] == 'value') + self.checks.append(kwargs['number'] == 123) + self.crawler.stats.inc_value('boolean_checks', 2) elif response.url.endswith('/general_without'): - self.checks.add(kwargs == {}) + self.checks.append(kwargs == {}) + self.crawler.stats.inc_value('boolean_checks') def parse_no_kwargs(self, response): - pass + self.checks.append(response.url.endswith('/no_kwargs')) + self.crawler.stats.inc_value('boolean_checks') class FollowAllSpider(MetaSpider): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3879a017c..9a39b8cb4 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -27,7 +27,8 @@ class CrawlTestCase(TestCase): def test_callback_kwargs(self): crawler = self.runner.create_crawler(KeywordArgumentsSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.checks, set([True])) + self.assertTrue(all(crawler.spider.checks)) + self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks')) @defer.inlineCallbacks def test_follow_all(self): From 6760bca74b1f51ce83ed73318d5ddcef03c9d129 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 Mar 2019 22:32:45 +0000 Subject: [PATCH 0335/1853] Rename Request.kwargs to Request.cb_kwargs --- docs/topics/request-response.rst | 16 ++++++++-------- scrapy/core/scraper.py | 4 +++- scrapy/http/request/__init__.py | 14 +++++++------- scrapy/http/response/__init__.py | 4 ++-- scrapy/http/response/text.py | 4 ++-- scrapy/utils/reqser.py | 4 ++-- tests/spiders.py | 6 +++--- tests/test_http_request.py | 8 ++++---- tests/test_utils_reqser.py | 4 ++-- 9 files changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d12766676..b3f849540 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs]) +.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -126,8 +126,8 @@ Request objects :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list - :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback. - :type kwargs: dict + :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback. + :type cb_kwargs: dict .. attribute:: Request.url @@ -168,7 +168,7 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. - .. attribute:: Request.kwargs + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents will be passed to the Request's callback as keyword arguments. It is empty @@ -177,7 +177,7 @@ Request objects This dict is `shallow copied`_ when the request is cloned using the ``copy()`` or ``replace()`` methods, and can also be accessed, in your - spider, from the ``response.kwargs`` attribute. + spider, from the ``response.cb_kwargs`` attribute. .. _shallow copied: https://docs.python.org/2/library/copy.html @@ -216,7 +216,7 @@ Example:: In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. The following two examples show how to achieve this by using the -:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively:: +:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively:: def parse_page1(self, response): item = MyItem() @@ -238,8 +238,8 @@ The following two examples show how to achieve this by using the item['main_url'] = response.url request = scrapy.Request("http://www.example.com/some_page.html", callback=self.parse_page2) - request.kwargs['item'] = item - request.kwargs['foo'] = 'bar' + request.cb_kwargs['item'] = item + request.cb_kwargs['foo'] = 'bar' yield request def parse_page2(self, response, item, foo): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7981ce231..08dd1acc5 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -143,7 +143,9 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs) + dfd.addCallbacks(callback=request.callback or spider.parse, + errback=request.errback, + callbackKeywords=request.cb_kwargs) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index c016eb727..f5935c4ef 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -18,7 +18,7 @@ class Request(object_ref): def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, kwargs=None): + dont_filter=False, errback=None, flags=None, cb_kwargs=None): self._encoding = encoding # this one has to be set first self.method = str(method).upper() @@ -40,14 +40,14 @@ class Request(object_ref): self.dont_filter = dont_filter self._meta = dict(meta) if meta else None - self._kwargs = dict(kwargs) if kwargs else None + self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) @property - def kwargs(self): - if self._kwargs is None: - self._kwargs = {} - return self._kwargs + def cb_kwargs(self): + if self._cb_kwargs is None: + self._cb_kwargs = {} + return self._cb_kwargs @property def meta(self): @@ -99,7 +99,7 @@ class Request(object_ref): given new values. """ for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']: + 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 99b04a26e..b0a526b72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -106,7 +106,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -133,4 +133,4 @@ class Response(object_ref): priority=priority, dont_filter=dont_filter, errback=errback, - kwargs=kwargs) + cb_kwargs=cb_kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 2039621b3..339913d4e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -123,7 +123,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -155,7 +155,7 @@ class TextResponse(Response): priority=priority, dont_filter=dont_filter, errback=errback, - kwargs=kwargs, + cb_kwargs=cb_kwargs, ) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d537057b1..e7016b92a 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -33,7 +33,7 @@ def request_to_dict(request, spider=None): 'priority': request.priority, 'dont_filter': request.dont_filter, 'flags': request.flags, - 'kwargs': request.kwargs, + 'cb_kwargs': request.cb_kwargs, } if type(request) is not Request: d['_class'] = request.__module__ + '.' + request.__class__.__name__ @@ -66,7 +66,7 @@ def request_from_dict(d, spider=None): priority=d['priority'], dont_filter=d['dont_filter'], flags=d.get('flags'), - kwargs=d.get('kwargs'), + cb_kwargs=d.get('cb_kwargs'), ) diff --git a/tests/spiders.py b/tests/spiders.py index 7b4707f62..a06985837 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -35,8 +35,8 @@ class KeywordArgumentsSpider(MockServerSpider): def start_requests(self): data = {'key': 'value', 'number': 123} - yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data) - yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data) + yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) + yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) @@ -47,7 +47,7 @@ class KeywordArgumentsSpider(MockServerSpider): yield response.follow( self.mockserver.url('/two'), self.parse_second, - kwargs={'new_key': 'new_value'}) + cb_kwargs={'new_key': 'new_value'}) def parse_second(self, response, new_key): self.checks.append(new_key == 'new_value') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 610893d8a..c1949a28c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -177,7 +177,7 @@ class RequestTest(unittest.TestCase): r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], callback=somecallback, errback=somecallback) r1.meta['foo'] = 'bar' - r1.kwargs['key'] = 'value' + r1.cb_kwargs['key'] = 'value' r2 = r1.copy() # make sure copy does not propagate callbacks @@ -190,9 +190,9 @@ class RequestTest(unittest.TestCase): assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" self.assertEqual(r1.flags, r2.flags) - # make sure kwargs dict is shallow copied - assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical" - self.assertEqual(r1.kwargs, r2.kwargs) + # make sure cb_kwargs dict is shallow copied + assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical" + self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 76de20f22..e1601b76b 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -26,7 +26,7 @@ class RequestSerializationTest(unittest.TestCase): encoding='latin-1', priority=20, meta={'a': 'b'}, - kwargs={'k': 'v'}, + cb_kwargs={'k': 'v'}, flags=['testFlag']) self._assert_serializes_ok(r, spider=self.spider) @@ -53,7 +53,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) - self.assertEqual(r1.kwargs, r2.kwargs) + self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) From 92bbc5290d2b381ea60d68442a887d1ba020874e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 16 Mar 2019 05:41:40 +0000 Subject: [PATCH 0336/1853] Rule.process_request - Renaming --- scrapy/spiders/crawl.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ad86fc19d..c01f75798 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -24,22 +24,22 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request_function = process_request or identity + self.process_request = process_request or identity if follow is None: self.follow = False if callback else True else: self.follow = follow - def process_request(self, request, response): + def _process_request(self, request, response): """ Wrapper around the request processing function to maintain backward compatibility with functions that do not take a Response object as parameter. """ - argcount = self.process_request_function.__code__.co_argcount - if getattr(self.process_request_function, '__self__', None): + argcount = self.process_request.__code__.co_argcount + if hasattr(self.process_request, '__self__'): argcount = argcount - 1 args = [request] if argcount == 1 else [request, response] - return self.process_request_function(*args) + return self.process_request(*args) class CrawlSpider(Spider): @@ -75,8 +75,8 @@ class CrawlSpider(Spider): links = rule.process_links(links) for link in links: seen.add(link) - r = self._build_request(n, link) - yield rule.process_request(r, response) + request = self._build_request(n, link) + yield rule._process_request(request, response) def _response_downloaded(self, response): rule = self._rules[response.meta['rule']] @@ -104,7 +104,7 @@ class CrawlSpider(Spider): for rule in self._rules: rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) - rule.process_request_function = get_method(rule.process_request_function) + rule.process_request = get_method(rule.process_request) @classmethod def from_crawler(cls, crawler, *args, **kwargs): From 044318920a463d2e04efe2a4c65d8f72d1b5ecb6 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sun, 17 Mar 2019 16:54:28 +0530 Subject: [PATCH 0337/1853] doc for creating custom cache storage backend. --- docs/topics/downloader-middleware.rst | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0d976077b..f913b059d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -496,6 +496,59 @@ In order to use this storage backend: .. _LevelDB: https://github.com/google/leveldb .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb +.. _httpcache-storage-custom: + +Implementing custom cache storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can implement custom cache storage backend by creating a Python class that +defines the methods described below. + +.. module:: scrapy.extensions.httpcache + +.. class:: CacheStorage + + .. method:: open_spider(spider) + + This method gets called after a spider has been opened for crawling. + + :param spider: the spider which has been opened + :type spider: :class:`~scrapy.spiders.Spider` object + + .. method:: close_spider(spider) + + This method gets called after a spider has been closed. + + :param spider: the spider which has been closed + :type spider: :class:`~scrapy.spiders.Spider` object + + .. method:: retrieve_response(spider, request) + + Returns response if present in cache, or ``None`` otherwise. + + :param spider: the spider which generated the request + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the request to find cached reponse for + :type request: :class:`~scrapy.http.Request` object + + .. method:: store_response(spider, request, response) + + Stores the given response in the cache. + + :param spider: the spider for which the response is intended + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: corresponding request the spider generated + :type request: :class:`~scrapy.http.Request` object + + :param response: the response to store in the cache + :type response: :class:`~scrapy.http.Response` object + +In order to use your storage backend, set: + +* :setting:`HTTPCACHE_STORAGE` to path of your custom storage class. + HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 282f24c510bdc6fff0e86ca16b7d681617af25ca Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 20 Mar 2019 18:46:22 +0530 Subject: [PATCH 0338/1853] Update form.py --- scrapy/http/request/form.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 9af2db5ff..03692f063 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, urlunsplit import lxml.html from parsel.selector import create_root_node @@ -33,12 +33,7 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(form_query_str) else: - url_split = urlsplit(self.url) - formdata_keys = set(dict(parse_qsl(form_query_str)).keys()) - items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] - url_query_str = _urlencode(items, self.encoding) - query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str - self._set_url(urlunsplit(url_split._replace(query = query))) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 4c89e53e684987c61857ef810374101440d5616c Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 20 Mar 2019 18:46:25 +0530 Subject: [PATCH 0339/1853] Update test_http_request.py --- tests/test_http_request.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5c30c9785..feea54a54 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,26 +275,17 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - #Both fragment and query in url data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertEqual(fs[b'c'], [b'3']) - #None of both in url data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #Duplicate GET arguments are preserved - data={'foo' : 'bar'} - fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) - self.assertEqual(fs[b'foo'], [b'bar']) - self.assertEqual(set(fs[b'a']), {b'1', b'2'}) - def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From 70aa5b1333a981b3b5eae70e29144c4c25fcfd4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Wed, 20 Mar 2019 15:32:20 +0100 Subject: [PATCH 0340/1853] Fix numeration --- docs/topics/selectors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index edc18f14d..282a585d4 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -436,7 +436,7 @@ The following examples show how these methods map to each other. >>> response.css('a::attr(href)').extract() ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] -2. ``Selector.get()`` is the same as ``Selector.extract()``:: +3. ``Selector.get()`` is the same as ``Selector.extract()``:: >>> response.css('a::attr(href)')[0].get() 'image1.html' From 821f5bb26077d7f9a6b2b1a72f210f81779f5393 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 3 Dec 2018 11:00:03 +0000 Subject: [PATCH 0341/1853] First implementation handle exception use O(N) instead of O(NlogN) here we have request as struct additional check for meptiness small performance improvement do not consume another request test number of responses mark requests back to 3 slots test case raise exceptions in case of missed meta add marks to requests and work only with your own requests only disk queue should obtain signals separate functions for slot rasd/write use signlas without variable stop crawler get signals in correct place logic test for download-aware priority queue update comment for structure ensure text type transform slot name to path use implicit structure use unicode type implicitly use real crawler add signals more slot accounting simple implementation of pop small slot accounting code no need for custom len function ability to call super in py27 add slots generic tests for downloader aware queue dummy implementation of crawler aware priority queue move common logic to base class rename class pass crawler to pqclass constructor do not copy quelib.PriorityQueue code add comment about new class remove obsolete function modify behaviour of queuelib.PriorityQueue to dodge very complex priority better way to get name remove obsolete commentary check boundaries function for priority convertion with known limits correct import path move file do not switch on by deffault as ip concurrency not supported set scheduler slot in case of empty slot use constant single place for added urls single place for constants use as default queue correct format for error text test migration from old version with on disk queue in these tests we have only two inflection points - jobdir and priority_queue_cls we do not need separate mock spider, use usual one do not rely on order of dict elements, imply order of list test round robiness of priority queue add comments and requirements for our magick function remove debug logging put queues into slot as we fabricate priorities we do not need special types anymore fabricate priority for priority queue more versatile priorities Scheduler class is not inflection point wrap correct types check for emptinees before initialization tests for new priority queue correct default type for startprios use exact values put common settings to base class test priorities for disk scheduler test dequeue for disk scheduler test length for disk scheduler setUp/tearDown methods for on disk schedulers new methods remove excessive line base class to handle scheduler creation correct method names test priorities deque test close scheduler on test end enqueue some requests test template for scheduler use downloader slot I/O implementation for RoundRobin queue round-robin implementation without I/O and slot detection wrappers for every disk queue class --- scrapy/core/downloader/__init__.py | 8 +- scrapy/core/queues.py | 15 -- scrapy/core/scheduler.py | 17 +- scrapy/pqueues.py | 246 ++++++++++++++++++++++ tests/test_scheduler.py | 315 +++++++++++++++++++++++++++++ 5 files changed, 578 insertions(+), 23 deletions(-) delete mode 100644 scrapy/core/queues.py create mode 100644 scrapy/pqueues.py create mode 100644 tests/test_scheduler.py diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 59c3ad074..4695d75f4 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -75,6 +75,8 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader(object): + DOWNLOAD_SLOT = 'download_slot' + def __init__(self, crawler): self.settings = crawler.settings self.signals = crawler.signals @@ -111,8 +113,8 @@ class Downloader(object): return key, self.slots[key] def _get_slot_key(self, request, spider): - if 'download_slot' in request.meta: - return request.meta['download_slot'] + if self.DOWNLOAD_SLOT in request.meta: + return request.meta[self.DOWNLOAD_SLOT] key = urlparse_cached(request).hostname or '' if self.ip_concurrency: @@ -122,7 +124,7 @@ class Downloader(object): def _enqueue_request(self, request, spider): key, slot = self._get_slot(request, spider) - request.meta['download_slot'] = key + request.meta[self.DOWNLOAD_SLOT] = key def _deactivate(response): slot.active.remove(request) diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py deleted file mode 100644 index 96d582fc7..000000000 --- a/scrapy/core/queues.py +++ /dev/null @@ -1,15 +0,0 @@ -import uuid -import os.path - - -def unique_files_queue(queue_class): - - class UniqueFilesQueue(queue_class): - def __init__(self, path): - path = path + "-" + uuid.uuid4().hex - while os.path.exists(path): - path = path + "-" + uuid.uuid4().hex - - super().__init__(path) - - return UniqueFilesQueue diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index eb790a67e..d40f3aa0c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class Scheduler(object): def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, - logunser=False, stats=None, pqclass=None): + logunser=False, stats=None, pqclass=None, crawler=None): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.pqclass = pqclass @@ -21,6 +21,7 @@ class Scheduler(object): self.mqclass = mqclass self.logunser = logunser self.stats = stats + self.crawler = crawler @classmethod def from_crawler(cls, crawler): @@ -32,14 +33,15 @@ class Scheduler(object): mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG')) return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, - stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass) + stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, + mqclass=mqclass, crawler=crawler) def has_pending_requests(self): return len(self) > 0 def open(self, spider): self.spider = spider - self.mqs = self.pqclass(self._newmq) + self.mqs = create_instance(self.pqclass, None, self.crawler, self._newmq) self.dqs = self._dq() if self.dqdir else None return self.df.open() @@ -111,7 +113,7 @@ class Scheduler(object): return self.mqclass() def _newdq(self, priority): - return self.dqclass(join(self.dqdir, 'p%s' % priority)) + return self.dqclass(join(self.dqdir, 'p%s' % (priority, ))) def _dq(self): activef = join(self.dqdir, 'active.json') @@ -120,7 +122,12 @@ class Scheduler(object): prios = json.load(f) else: prios = () - q = self.pqclass(self._newdq, startprios=prios) + + q = create_instance(self.pqclass, + None, + self.crawler, + self._newdq, + startprios=prios) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", {'queuesize': len(q)}, extra={'spider': self.spider}) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py new file mode 100644 index 000000000..75073b7a4 --- /dev/null +++ b/scrapy/pqueues.py @@ -0,0 +1,246 @@ +from collections import deque +import hashlib +import logging +from six import text_type +from six.moves.urllib.parse import urlparse + +from queuelib import PriorityQueue + +from scrapy.core.downloader import Downloader +from scrapy.http import Request +from scrapy.signals import request_reached_downloader, response_downloaded + + +logger = logging.getLogger(__name__) + + +SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT + + +def _get_from_request(request, key, default=None): + if isinstance(request, dict): + return request.get(key, default) + + if isinstance(request, Request): + return getattr(request, key, default) + + raise ValueError('Bad type of request "%s"' % (request.__class__, )) + + +def _scheduler_slot_read(request, default=None): + meta = _get_from_request(request, 'meta', dict()) + slot = meta.get(SCHEDULER_SLOT_META_KEY, default) + return slot + + +def _scheduler_slot_write(request, slot): + meta = _get_from_request(request, 'meta', None) + if not isinstance(meta, dict): + raise ValueError('No meta attribute in %s' % (request, )) + meta[SCHEDULER_SLOT_META_KEY] = slot + + +def _scheduler_slot(request): + + slot = _scheduler_slot_read(request, None) + if slot is None: + url = _get_from_request(request, 'url') + slot = urlparse(url).hostname or '' + _scheduler_slot_write(request, slot) + + return slot + + +def _pathable(x): + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x]) + + """ + as we replace some letters we can get collision for different slots + add we add unique part + """ + unique_slot = hashlib.md5(x.encode('utf8')).hexdigest() + + return '-'.join([pathable_slot, unique_slot]) + + +class PrioritySlot: + __slots__ = ('priority', 'slot') + + def __init__(self, priority=0, slot=None): + self.priority = priority + self.slot = slot + + def __hash__(self): + return hash((self.priority, self.slot)) + + def __eq__(self, other): + return (self.priority, self.slot) == (other.priority, other.slot) + + def __lt__(self, other): + return (self.priority, self.slot) < (other.priority, other.slot) + + def __str__(self): + return '_'.join([text_type(self.priority), _pathable(text_type(self.slot))]) + + +class PriorityAsTupleQueue(PriorityQueue): + """ + Python structures is not directly (de)serialized (to)from json. + We need this modified queue to transform custom structure (from)to + json serializable structures + """ + def __init__(self, qfactory, startprios=()): + + super(PriorityAsTupleQueue, self).__init__( + qfactory, + [PrioritySlot(priority=p[0], slot=p[1]) for p in startprios] + ) + + def close(self): + startprios = super(PriorityAsTupleQueue, self).close() + return [(s.priority, s.slot) for s in startprios] + + def is_empty(self): + return not self.queues or len(self) == 0 + + +class SlotBasedPriorityQueue(object): + + def __init__(self, qfactory, startprios={}): + self.pqueues = dict() # slot -> priority queue + self.qfactory = qfactory # factory for creating new internal queues + + if not startprios: + return + + if not isinstance(startprios, dict): + raise ValueError("Looks like your priorities file malforfemed. " + "Possible reason: You run scrapy with previous " + "version. Interrupted it. Updated scrapy. And " + "run again.") + + for slot, prios in startprios.items(): + self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory, prios) + + def pop_slot(self, slot): + queue = self.pqueues[slot] + request = queue.pop() + is_empty = queue.is_empty() + if is_empty: + del self.pqueues[slot] + + return request, is_empty + + def push_slot(self, request, priority): + slot = _scheduler_slot(request) + is_new = False + if slot not in self.pqueues: + is_new = True + self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory) + self.pqueues[slot].push(request, PrioritySlot(priority=priority, slot=slot)) + return slot, is_new + + def close(self): + startprios = dict() + for slot, queue in self.pqueues.items(): + prios = queue.close() + startprios[slot] = prios + self.pqueues.clear() + return startprios + + def __len__(self): + return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + + +class RoundRobinPriorityQueue(SlotBasedPriorityQueue): + + def __init__(self, qfactory, startprios={}): + super(RoundRobinPriorityQueue, self).__init__(qfactory, startprios) + self._slots = deque() + for slot in self.pqueues: + self._slots.append(slot) + + def push(self, request, priority): + slot, is_new = self.push_slot(request, priority) + if is_new: + self._slots.append(slot) + + def pop(self): + if not self._slots: + return + + slot = self._slots.popleft() + request, is_empty = self.pop_slot(slot) + + if not is_empty: + self._slots.append(slot) + + return request + + def close(self): + self._slots.clear() + return super(RoundRobinPriorityQueue, self).close() + + +class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): + + _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' + + @classmethod + def from_crawler(cls, crawler, qfactory, startprios={}): + return cls(crawler, qfactory, startprios) + + def __init__(self, crawler, qfactory, startprios={}): + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) + self._slots = {slot: 0 for slot in self.pqueues} + crawler.signals.connect(self.on_response_download, + signal=response_downloaded) + crawler.signals.connect(self.on_request_reached_downloader, + signal=request_reached_downloader) + + def mark(self, request): + meta = _get_from_request(request, 'meta', None) + if not isinstance(meta, dict): + raise ValueError('No meta attribute in %s' % (request, )) + meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) + + def check_mark(self, request): + return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) + + def pop(self): + slots = [(d, s) for s,d in self._slots.items() if s in self.pqueues] + + if not slots: + return + + slot = min(slots)[1] + request, _ = self.pop_slot(slot) + self.mark(request) + return request + + def push(self, request, priority): + slot, _ = self.push_slot(request, priority) + if slot not in self._slots: + self._slots[slot] = 0 + + def on_response_download(self, response, request, spider): + if not self.check_mark(request): + return + + slot = _scheduler_slot_read(request) + if slot not in self._slots or self._slots[slot] <= 0: + raise ValueError('Get response for wrong slot "%s"' % (slot, )) + self._slots[slot] = self._slots[slot] - 1 + if self._slots[slot] == 0 and slot not in self.pqueues: + del self._slots[slot] + + def on_request_reached_downloader(self, request, spider): + if not self.check_mark(request): + return + + slot = _scheduler_slot_read(request) + self._slots[slot] = self._slots.get(slot, 0) + 1 + + def close(self): + self._slots.clear() + return super(DownloaderAwarePriorityQueue, self).close() diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 000000000..fd86e8d8c --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,315 @@ +import contextlib +import shutil +import tempfile +import unittest + +from scrapy.crawler import Crawler +from scrapy.core.scheduler import Scheduler +from scrapy.http import Request +from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write +from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.spiders import Spider + +class MockCrawler(Crawler): + def __init__(self, priority_queue_cls, jobdir): + + settings = dict(LOG_UNSERIALIZABLE_REQUESTS=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter') + super(MockCrawler, self).__init__(Spider, settings) + + +class SchedulerHandler: + priority_queue_cls = None + jobdir = None + + def create_scheduler(self): + self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir) + self.scheduler = Scheduler.from_crawler(self.mock_crawler) + self.spider = Spider(name='spider') + self.scheduler.open(self.spider) + + def close_scheduler(self): + self.scheduler.close('finished') + self.mock_crawler.stop() + + def setUp(self): + self.create_scheduler() + + def tearDown(self): + self.close_scheduler() + + +_PRIORITIES = [("http://foo.com/a", -2), + ("http://foo.com/d", 1), + ("http://foo.com/b", -1), + ("http://foo.com/c", 0), + ("http://foo.com/e", 2)] + + +_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"} + + +class BaseSchedulerInMemoryTester(SchedulerHandler): + def test_length(self): + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(_URLS)) + + def test_dequeue(self): + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + urls = set() + while self.scheduler.has_pending_requests(): + urls.add(self.scheduler.next_request().url) + + self.assertEqual(urls, _URLS) + + def test_dequeue_priorities(self): + for url, priority in _PRIORITIES: + self.scheduler.enqueue_request(Request(url, priority=priority)) + + priorities = list() + while self.scheduler.has_pending_requests(): + priorities.append(self.scheduler.next_request().priority) + + self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + + +class BaseSchedulerOnDiskTester(SchedulerHandler): + + def setUp(self): + self.jobdir = tempfile.mkdtemp() + self.create_scheduler() + + def tearDown(self): + self.close_scheduler() + + shutil.rmtree(self.jobdir) + self.jobdir = None + + def test_length(self): + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + self.close_scheduler() + self.create_scheduler() + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(_URLS)) + + def test_dequeue(self): + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + self.close_scheduler() + self.create_scheduler() + + urls = set() + while self.scheduler.has_pending_requests(): + urls.add(self.scheduler.next_request().url) + + self.assertEqual(urls, _URLS) + + def test_dequeue_priorities(self): + for url, priority in _PRIORITIES: + self.scheduler.enqueue_request(Request(url, priority=priority)) + + self.close_scheduler() + self.create_scheduler() + + priorities = list() + while self.scheduler.has_pending_requests(): + priorities.append(self.scheduler.next_request().priority) + + self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + + +class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): + priority_queue_cls = 'queuelib.PriorityQueue' + + +class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): + priority_queue_cls = 'queuelib.PriorityQueue' + + +_SLOTS = [("http://foo.com/a", 'a'), + ("http://foo.com/b", 'a'), + ("http://foo.com/c", 'b'), + ("http://foo.com/d", 'b'), + ("http://foo.com/e", 'c'), + ("http://foo.com/f", 'c')] + + +class TestSchedulerWithRoundRobinInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + + def test_round_robin(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + slots = list() + while self.scheduler.has_pending_requests(): + slots.append(_scheduler_slot_read(self.scheduler.next_request())) + + for i in range(0, len(_SLOTS), 2): + self.assertNotEqual(slots[i], slots[i+1]) + + def test_is_meta_set(self): + url = "http://foo.com/a" + request = Request(url) + if _scheduler_slot_read(request): + _scheduler_slot_write(request, None) + self.scheduler.enqueue_request(request) + self.assertIsNotNone(_scheduler_slot_read(request, None), None) + + +class TestSchedulerWithRoundRobinOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + + def test_round_robin(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + self.close_scheduler() + self.create_scheduler() + + slots = list() + while self.scheduler.has_pending_requests(): + slots.append(_scheduler_slot_read(self.scheduler.next_request())) + + for i in range(0, len(_SLOTS), 2): + self.assertNotEqual(slots[i], slots[i+1]) + + def test_is_meta_set(self): + url = "http://foo.com/a" + request = Request(url) + if _scheduler_slot_read(request): + _scheduler_slot_write(request, None) + self.scheduler.enqueue_request(request) + + self.close_scheduler() + self.create_scheduler() + + self.assertIsNotNone(_scheduler_slot_read(request, None), None) + + +@contextlib.contextmanager +def mkdtemp(): + dir = tempfile.mkdtemp() + try: + yield dir + finally: + shutil.rmtree(dir) + + +def _migration(): + + with mkdtemp() as tmp_dir: + prev_scheduler_handler = SchedulerHandler() + prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue' + prev_scheduler_handler.jobdir = tmp_dir + + prev_scheduler_handler.create_scheduler() + for url in _URLS: + prev_scheduler_handler.scheduler.enqueue_request(Request(url)) + prev_scheduler_handler.close_scheduler() + + next_scheduler_handler = SchedulerHandler() + next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + next_scheduler_handler.jobdir = tmp_dir + + next_scheduler_handler.create_scheduler() + + +class TestMigration(unittest.TestCase): + def test_migration(self): + self.assertRaises(ValueError, _migration) + + +class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + + def test_logic(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + slots = list() + requests = list() + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + slots.append(_scheduler_slot_read(request)) + self.mock_crawler.signals.send_catch_log( + signal=request_reached_downloader, + request=request, + spider=self.spider + ) + requests.append(request) + self.assertEqual(len(slots), len(_SLOTS)) + + for request in requests: + self.mock_crawler.signals.send_catch_log(signal=response_downloaded, + request=request, + response=None, + spider=self.spider) + + unique_slots = len(set(s for _, s in _SLOTS)) + for i in range(0, len(_SLOTS), unique_slots): + part = slots[i:i + unique_slots] + self.assertEqual(len(part), len(set(part))) + + +class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + def test_logic(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + self.close_scheduler() + self.create_scheduler() + + slots = list() + requests = list() + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + slots.append(_scheduler_slot_read(request)) + self.mock_crawler.signals.send_catch_log( + signal=request_reached_downloader, + request=request, + spider=self.spider + ) + requests.append(request) + + self.assertEqual(self.scheduler.mqs._slots, {}) + self.assertEqual(len(slots), len(_SLOTS)) + + for request in requests: + self.mock_crawler.signals.send_catch_log(signal=response_downloaded, + request=request, + response=None, + spider=self.spider) + + unique_slots = len(set(s for _, s in _SLOTS)) + for i in range(0, len(_SLOTS), unique_slots): + part = slots[i:i + unique_slots] + self.assertEqual(len(part), len(set(part))) From afdb69ea6daac8bd4f580d6c20bf9e93b741957b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 16:36:05 +0100 Subject: [PATCH 0342/1853] Add a troubleshooting section to the installation instructions Its initial content covers the workaround for #2473. --- docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4a9aa3cfb..daec7fcb7 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. -For more detailed and platform specifics instructions, read on. +For more detailed and platform specifics instructions, as well as +troubleshooting information, read on. Things that are good to know @@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency. To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +.. _intro-install-troubleshooting: + +Troubleshooting +=============== + +AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' +---------------------------------------------------------------- + +After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an +exception with the following traceback:: + + […] + File "[…]/site-packages/twisted/protocols/tls.py", line 63, in + from twisted.internet._sslverify import _setAcceptableProtocols + File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in + TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1, + AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' + +The reason you get this exception is that your system or virtual environment +has a version of pyOpenSSL that your version of Twisted does not support. + +To install a version of pyOpenSSL that your version of Twisted supports, +reinstall Twisted with the :code:`tls` extra option:: + + pip install twisted[tls] + +For details, see `Issue #2473 `_. + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ From 9c314800e4b195df41e5c0aba0d9ffe4bcffec8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 17:14:10 +0100 Subject: [PATCH 0343/1853] Document the SCRAPY_PROJECT environment variable Fixes #1109 --- docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ef9c45196..97f8311de 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`) -* ``SCRAPY_PROJECT`` +* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`) * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) .. _topics-project-structure: @@ -71,6 +71,33 @@ the project settings. Here is an example:: [settings] default = myproject.settings +.. _topics-project-envvar: + +Sharing the root directory between projects +=========================================== + +A project root directory, the one that contains the ``scrapy.cfg``, may be +shared by multiple Scrapy projects, each with its own settings module. + +In that case, you must define one or more aliases for those settings modules +under ``[settings]`` in your ``scrapy.cfg`` file:: + + [settings] + default = myproject1.settings + project1 = myproject1.settings + project2 = myproject2.settings + +By default, the ``scrapy`` command-line tool will use the ``default`` settings. +Use the ``SCRAPY_PROJECT`` environment variable to specify a different project +for ``scrapy`` to use:: + + $ scrapy settings --get BOT_NAME + Project 1 Bot + $ export SCRAPY_PROJECT=project2 + $ scrapy settings --get BOT_NAME + Project 2 Bot + + Using the ``scrapy`` tool ========================= From f56079f6c71a77c1f70510cf291cd808617933cd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Dec 2018 10:02:42 +0000 Subject: [PATCH 0344/1853] Test cleanups PEP8 fixes no need to close implicitly do not use pytest need to put it into class remove round-robin queue additional check for empty queue use pytest tmpdir fixture --- scrapy/pqueues.py | 50 ++++------------ tests/test_scheduler.py | 128 ++++++++++++---------------------------- 2 files changed, 50 insertions(+), 128 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75073b7a4..287a8de35 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,4 +1,3 @@ -from collections import deque import hashlib import logging from six import text_type @@ -71,16 +70,17 @@ class PrioritySlot: self.slot = slot def __hash__(self): - return hash((self.priority, self.slot)) + return hash((self.priority, self.slot)) def __eq__(self, other): - return (self.priority, self.slot) == (other.priority, other.slot) + return (self.priority, self.slot) == (other.priority, other.slot) def __lt__(self, other): - return (self.priority, self.slot) < (other.priority, other.slot) + return (self.priority, self.slot) < (other.priority, other.slot) def __str__(self): - return '_'.join([text_type(self.priority), _pathable(text_type(self.slot))]) + return '_'.join([text_type(self.priority), + _pathable(text_type(self.slot))]) class PriorityAsTupleQueue(PriorityQueue): @@ -135,9 +135,10 @@ class SlotBasedPriorityQueue(object): slot = _scheduler_slot(request) is_new = False if slot not in self.pqueues: - is_new = True self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory) - self.pqueues[slot].push(request, PrioritySlot(priority=priority, slot=slot)) + queue = self.pqueues[slot] + is_new = queue.is_empty() + queue.push(request, PrioritySlot(priority=priority, slot=slot)) return slot, is_new def close(self): @@ -152,36 +153,6 @@ class SlotBasedPriorityQueue(object): return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 -class RoundRobinPriorityQueue(SlotBasedPriorityQueue): - - def __init__(self, qfactory, startprios={}): - super(RoundRobinPriorityQueue, self).__init__(qfactory, startprios) - self._slots = deque() - for slot in self.pqueues: - self._slots.append(slot) - - def push(self, request, priority): - slot, is_new = self.push_slot(request, priority) - if is_new: - self._slots.append(slot) - - def pop(self): - if not self._slots: - return - - slot = self._slots.popleft() - request, is_empty = self.pop_slot(slot) - - if not is_empty: - self._slots.append(slot) - - return request - - def close(self): - self._slots.clear() - return super(RoundRobinPriorityQueue, self).close() - - class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' @@ -191,7 +162,8 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return cls(crawler, qfactory, startprios) def __init__(self, crawler, qfactory, startprios={}): - super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, + startprios) self._slots = {slot: 0 for slot in self.pqueues} crawler.signals.connect(self.on_response_download, signal=response_downloaded) @@ -208,7 +180,7 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) def pop(self): - slots = [(d, s) for s,d in self._slots.items() if s in self.pqueues] + slots = [(d, s) for s, d in self._slots.items() if s in self.pqueues] if not slots: return diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index fd86e8d8c..e1cf5842d 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,4 +1,3 @@ -import contextlib import shutil import tempfile import unittest @@ -10,15 +9,18 @@ from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write from scrapy.signals import request_reached_downloader, response_downloaded from scrapy.spiders import Spider + class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict(LOG_UNSERIALIZABLE_REQUESTS=False, - SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', - SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter') + settings = dict( + LOG_UNSERIALIZABLE_REQUESTS=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' + ) super(MockCrawler, self).__init__(Spider, settings) @@ -82,7 +84,8 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + self.assertEqual(priorities, + sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) class BaseSchedulerOnDiskTester(SchedulerHandler): @@ -134,7 +137,8 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + self.assertEqual(priorities, + sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): @@ -153,75 +157,15 @@ _SLOTS = [("http://foo.com/a", 'a'), ("http://foo.com/f", 'c')] -class TestSchedulerWithRoundRobinInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): - priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' +class TestMigration(unittest.TestCase): - def test_round_robin(self): - for url, slot in _SLOTS: - request = Request(url) - _scheduler_slot_write(request, slot) - self.scheduler.enqueue_request(request) + def setUp(self): + self.tmpdir = tempfile.mkdtemp() - slots = list() - while self.scheduler.has_pending_requests(): - slots.append(_scheduler_slot_read(self.scheduler.next_request())) + def tearDown(self): + shutil.rmtree(self.tmpdir) - for i in range(0, len(_SLOTS), 2): - self.assertNotEqual(slots[i], slots[i+1]) - - def test_is_meta_set(self): - url = "http://foo.com/a" - request = Request(url) - if _scheduler_slot_read(request): - _scheduler_slot_write(request, None) - self.scheduler.enqueue_request(request) - self.assertIsNotNone(_scheduler_slot_read(request, None), None) - - -class TestSchedulerWithRoundRobinOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): - priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' - - def test_round_robin(self): - for url, slot in _SLOTS: - request = Request(url) - _scheduler_slot_write(request, slot) - self.scheduler.enqueue_request(request) - - self.close_scheduler() - self.create_scheduler() - - slots = list() - while self.scheduler.has_pending_requests(): - slots.append(_scheduler_slot_read(self.scheduler.next_request())) - - for i in range(0, len(_SLOTS), 2): - self.assertNotEqual(slots[i], slots[i+1]) - - def test_is_meta_set(self): - url = "http://foo.com/a" - request = Request(url) - if _scheduler_slot_read(request): - _scheduler_slot_write(request, None) - self.scheduler.enqueue_request(request) - - self.close_scheduler() - self.create_scheduler() - - self.assertIsNotNone(_scheduler_slot_read(request, None), None) - - -@contextlib.contextmanager -def mkdtemp(): - dir = tempfile.mkdtemp() - try: - yield dir - finally: - shutil.rmtree(dir) - - -def _migration(): - - with mkdtemp() as tmp_dir: + def _migration(self, tmp_dir): prev_scheduler_handler = SchedulerHandler() prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue' prev_scheduler_handler.jobdir = tmp_dir @@ -232,18 +176,18 @@ def _migration(): prev_scheduler_handler.close_scheduler() next_scheduler_handler = SchedulerHandler() - next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' next_scheduler_handler.jobdir = tmp_dir next_scheduler_handler.create_scheduler() - -class TestMigration(unittest.TestCase): def test_migration(self): - self.assertRaises(ValueError, _migration) + with self.assertRaises(ValueError): + self._migration(self.tmpdir) -class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): +class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, + unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): @@ -266,10 +210,12 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unit self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log(signal=response_downloaded, - request=request, - response=None, - spider=self.spider) + self.mock_crawler.signals.send_catch_log( + signal=response_downloaded, + request=request, + response=None, + spider=self.spider + ) unique_slots = len(set(s for _, s in _SLOTS)) for i in range(0, len(_SLOTS), unique_slots): @@ -277,8 +223,10 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unit self.assertEqual(len(part), len(set(part))) -class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): +class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, + unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + def test_logic(self): for url, slot in _SLOTS: request = Request(url) @@ -304,10 +252,12 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log(signal=response_downloaded, - request=request, - response=None, - spider=self.spider) + self.mock_crawler.signals.send_catch_log( + signal=response_downloaded, + request=request, + response=None, + spider=self.spider + ) unique_slots = len(set(s for _, s in _SLOTS)) for i in range(0, len(_SLOTS), unique_slots): From 7efba101946af93397ec3c2323b920644e20ce04 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 10 Dec 2018 14:44:15 +0800 Subject: [PATCH 0345/1853] remove "sudo: false" now that travis no longer supports it https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4218d13bf..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: python -sudo: false branches: only: - master From 0e06b9a81672ec432d2fccc3cbacc823ea47b656 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:35:18 +0000 Subject: [PATCH 0346/1853] use urlparse_cached where it is possible --- scrapy/pqueues.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 287a8de35..ff7ec8c8a 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -8,6 +8,7 @@ from queuelib import PriorityQueue from scrapy.core.downloader import Downloader from scrapy.http import Request from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) @@ -41,11 +42,26 @@ def _scheduler_slot_write(request, slot): def _scheduler_slot(request): - slot = _scheduler_slot_read(request, None) - if slot is None: - url = _get_from_request(request, 'url') + if isinstance(request, dict): + meta = request.get('meta', dict()) + elif isinstance(request, Request): + meta = request.meta + else: + raise ValueError('Bad type of request "%s"' % (request.__class__, )) + + slot = meta.get(SCHEDULER_SLOT_META_KEY, None) + + if slot is not None: + return slot + + if isinstance(request, dict): + url = request.get('url', None) slot = urlparse(url).hostname or '' - _scheduler_slot_write(request, slot) + elif isinstance(request, Request): + url = request.url + slot = urlparse_cached(request).hostname or '' + + meta[SCHEDULER_SLOT_META_KEY] = slot return slot From 484927b08caff66ea622f8553468c831154df30a Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:38:28 +0000 Subject: [PATCH 0347/1853] less complex implementation --- scrapy/pqueues.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index ff7ec8c8a..538678345 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -28,16 +28,11 @@ def _get_from_request(request, key, default=None): def _scheduler_slot_read(request, default=None): - meta = _get_from_request(request, 'meta', dict()) - slot = meta.get(SCHEDULER_SLOT_META_KEY, default) - return slot + return request.meta.get(SCHEDULER_SLOT_META_KEY, default) def _scheduler_slot_write(request, slot): - meta = _get_from_request(request, 'meta', None) - if not isinstance(meta, dict): - raise ValueError('No meta attribute in %s' % (request, )) - meta[SCHEDULER_SLOT_META_KEY] = slot + request.meta[SCHEDULER_SLOT_META_KEY] = slot def _scheduler_slot(request): From 6af964cc0b47c570e035a3486b9f8aebd349bd84 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:54:24 +0000 Subject: [PATCH 0348/1853] common indentation for comment --- scrapy/pqueues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 538678345..31e90ff12 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -96,9 +96,9 @@ class PrioritySlot: class PriorityAsTupleQueue(PriorityQueue): """ - Python structures is not directly (de)serialized (to)from json. - We need this modified queue to transform custom structure (from)to - json serializable structures + Python structures is not directly (de)serialized (to)from json. + We need this modified queue to transform custom structure (from)to + json serializable structures """ def __init__(self, qfactory, startprios=()): From a46613afa8acd136f4ba62df2ced2f3c87679512 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:55:06 +0000 Subject: [PATCH 0349/1853] use regular comments --- scrapy/pqueues.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 31e90ff12..75fc198d0 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -64,10 +64,8 @@ def _scheduler_slot(request): def _pathable(x): pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x]) - """ - as we replace some letters we can get collision for different slots - add we add unique part - """ + # as we replace some letters we can get collision for different slots + # add we add unique part unique_slot = hashlib.md5(x.encode('utf8')).hexdigest() return '-'.join([pathable_slot, unique_slot]) From a23e1894b3a09e1daf49dd9592546b2d21bc9a72 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 16:18:34 +0000 Subject: [PATCH 0350/1853] Fix boto problem another way to fix boto problem Revert "fix for travis ci based on https://github.com/boto/boto/issues/3717" This reverts commit 150d2564ff0ea994652da7f5be333d72e0b38d93. fix for travis ci based on https://github.com/boto/boto/issues/3717 --- tests/requirements-py2.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index 790f29d34..f5bcfda60 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -11,3 +11,4 @@ testfixtures # optional for shell wrapper tests bpython ipython<6.0 +google-compute-engine From d970be64cc47c382bd615cd547e7e94c17e27b48 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 17 Dec 2018 13:52:11 +0000 Subject: [PATCH 0351/1853] Integration test integration testing only everything is working, not logic of PQ use method create slot attribute in constructor corect class for test case stop crawler in teardown method use class correct entity naming python 2 adaptation integration test with crawler and spider --- tests/test_scheduler.py | 46 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index e1cf5842d..9bdc82b30 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,12 +2,17 @@ import shutil import tempfile import unittest +from twisted.internet import defer +from twisted.trial.unittest import TestCase + from scrapy.crawler import Crawler from scrapy.core.scheduler import Scheduler from scrapy.http import Request from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write from scrapy.signals import request_reached_downloader, response_downloaded from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer class MockCrawler(Crawler): @@ -223,6 +228,13 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, self.assertEqual(len(part), len(set(part))) +def _is_slots_unique(base_slots, result_slots): + unique_slots = len(set(s for _, s in base_slots)) + for i in range(0, len(result_slots), unique_slots): + part = result_slots[i:i + unique_slots] + assert len(part) == len(set(part)) + + class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' @@ -259,7 +271,33 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, spider=self.spider ) - unique_slots = len(set(s for _, s in _SLOTS)) - for i in range(0, len(_SLOTS), unique_slots): - part = slots[i:i + unique_slots] - self.assertEqual(len(part), len(set(part))) + _is_slots_unique(_SLOTS, slots) + + +class StartUrlsSpider(Spider): + + def __init__(self, start_urls): + self.start_urls = start_urls + + +class TestIntegrationWithDownloaderAwareOnDisk(TestCase): + def setUp(self): + self.crawler = get_crawler( + StartUrlsSpider, + {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'} + ) + + @defer.inlineCallbacks + def tearDown(self): + yield self.crawler.stop() + + @defer.inlineCallbacks + def test_integration_downloader_aware_priority_queue(self): + with MockServer() as mockserver: + + url = mockserver.url("/status?n=200", is_secure=False) + slots = [url] * 6 + yield self.crawler.crawl(slots) + self.assertEqual(self.crawler.stats.get_value('downloader/response_count'), + len(slots)) From 7d3175ac8433f964ebbb80ebd67f9899cf059100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:23:23 -0300 Subject: [PATCH 0352/1853] Fix boto import error under Jessie testing environment --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 08b0bf119..a201f97b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,11 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [ "$TOXENV" = "jessie" ]; then + # Not used directly but allows boto GCE plugins to load. + # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + pip install google-compute-engine + fi - pip install -U tox twine wheel codecov script: tox From 6ff2574c277ba1eda31fb43f86f43d5b7b4bef09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:39:29 -0300 Subject: [PATCH 0353/1853] Needs to be installed within tox env --- .travis.yml | 5 ----- tox.ini | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a201f97b1..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,11 +42,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [ "$TOXENV" = "jessie" ]; then - # Not used directly but allows boto GCE plugins to load. - # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 - pip install google-compute-engine - fi - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e5543fe2a..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -51,6 +51,9 @@ deps = cssselect==0.9.1 zope.interface==4.1.1 -rtests/requirements-py2.txt +# Not used directly but allows boto GCE plugins to load. +# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + google-compute-engine==2.8.12 [testenv:trunk] basepython = python2.7 From 4163a7a1c7ac11c8d4db70f371c26181b90d8dfd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 21 Dec 2018 09:10:32 +0000 Subject: [PATCH 0354/1853] no need for this --- tests/requirements-py2.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index f5bcfda60..790f29d34 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -11,4 +11,3 @@ testfixtures # optional for shell wrapper tests bpython ipython<6.0 -google-compute-engine From 987c2ae4a964e45120c245235c9b0c49dc36b71f Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:13:09 +0000 Subject: [PATCH 0355/1853] test ip concurrency incompatibility with DAPQ --- tests/test_scheduler.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9bdc82b30..17b706bd7 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -301,3 +301,20 @@ class TestIntegrationWithDownloaderAwareOnDisk(TestCase): yield self.crawler.crawl(slots) self.assertEqual(self.crawler.stats.get_value('downloader/response_count'), len(slots)) + + +class TestIncompatibility(unittest.TestCase): + + def _incompatible(self): + settings = dict( + SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', + CONCURRENT_REQUESTS_PER_IP=1 + ) + crawler = Crawler(Spider, settings) + scheduler = Scheduler.from_crawler(crawler) + spider = Spider(name='spider') + scheduler.open(spider) + + def test_incompatibility(self): + with self.assertRaises(ValueError): + self._incompatible() From 8e8ce301b1a56e40f7e9c322a7b73b8dcfcefc43 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:14:09 +0000 Subject: [PATCH 0356/1853] check CONCURRENT_REQUESTS_PER_IP is not set --- scrapy/pqueues.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75fc198d0..d9effc9d1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -171,6 +171,14 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return cls(crawler, qfactory, startprios) def __init__(self, crawler, qfactory, startprios={}): + ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' + ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) + + if ip_concurrency > 0: + raise ValueError('"%s" does not support %s=%d' % (self.__class__, + ip_concurrency_key, + ip_concurrency)) + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) self._slots = {slot: 0 for slot in self.pqueues} From 338b78d796de6c93af0f4bcb762f82f5a14b87cd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:44:20 +0000 Subject: [PATCH 0357/1853] Add documentation add section to broad-crawl topic reword in accord with broad-crawl topic add documentation for new priority queue --- docs/topics/broad-crawls.rst | 11 +++++++++++ docs/topics/settings.rst | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index eb02086dc..37f7a8748 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -39,6 +39,17 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. +Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` +============================================== + +Default scrapy's scheduler priority queue is ``'queuelib.PriorityQueue'``. +It works best during single domain crawl. And it does not work well with crawling +many different domains in parallel + +To apply recommended priority queue use:: + + SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + Increase concurrency ==================== diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 47b6cf13d..7b9ff7e39 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1144,7 +1144,12 @@ SCHEDULER_PRIORITY_QUEUE ------------------------ Default: ``'queuelib.PriorityQueue'`` -Type of priority queue used by scheduler. +Type of priority queue used by scheduler. Another available type is +``scrapy.pqueues.DownloaderAwarePriorityQueue``. +``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than +``'queuelib.PriorityQueue'`` when you crawl many different domains in parallel. +But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. .. setting:: SPIDER_CONTRACTS From 2cb4dc32052c306568206f0997de4b4e53069efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 22 Mar 2019 09:50:11 -0300 Subject: [PATCH 0358/1853] Mentioning to use JSON API for ACLs --- docs/topics/settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90ae8fd93..fcdf31cac 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. .. setting:: FTP_PASSIVE_MODE @@ -1388,4 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search -.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file +.. _Google Cloud Storage: https://cloud.google.com/storage/ From 72cf190145196c3054b611eef7a0eef30ac63c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 14:46:07 +0100 Subject: [PATCH 0359/1853] Add a FAQ entry about name collisions --- docs/faq.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..8de680816 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -319,7 +319,18 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. +Running ``runspider`` I get ``error: No spider found in file: `` +-------------------------------------------------------------------------- + +This may happen if your Scrapy project has a spider module with a name that +conflicts with the name of one of the `Python standard library modules`_, such +as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed. +See :issue:`2680`. + +.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python package: https://pypi.org/ + .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file From 7c148fce5acc100f5f01719db374578bfca2512a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 15:40:16 +0100 Subject: [PATCH 0360/1853] Implement Item.deepcopy() --- docs/topics/items.rst | 44 +++++++++++++++++++++++++++++++++---------- scrapy/item.py | 8 ++++++++ tests/test_item.py | 8 ++++++++ 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index ae44aecd3..d744fd9ea 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -40,6 +40,7 @@ objects. Here is an example:: name = scrapy.Field() price = scrapy.Field() stock = scrapy.Field() + tags = scrapy.Field() last_updated = scrapy.Field(serializer=str) .. note:: Those familiar with `Django`_ will notice that Scrapy Items are @@ -155,19 +156,42 @@ To access all populated values, just use the typical `dict API`_:: >>> product.items() [('price', 1000), ('name', 'Desktop PC')] + +Copying items +------------- + +To copy an item, you must first decide whether you want a shallow copy or a +deep copy. + +If your item contains mutable_ values like lists or dictionaries, a shallow +copy will keep references to the same mutable values across all different +copies. + +.. _mutable: https://docs.python.org/glossary.html#term-mutable + +For example, if you have an item with a list of tags, and you create a shallow +copy of that item, both the original item and the copy have the same list of +tags. Adding a tag to the list of one of the items will add the tag to the +other item as well. + +If that is not the desired behavior, use a deep copy instead. + +See the `documentation of the copy module`_ for more information. + +.. _documentation of the copy module: https://docs.python.org/library/copy.html + +To create a shallow copy of an item, you can either call +:meth:`~scrapy.item.Item.copy` on an existing item +(``product2 = product.copy()``) or instantiate your item class from an existing +item (``product2 = Product(product)``). + +To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +(``product2 = product.deepcopy()``). + + Other common tasks ------------------ -Copying items:: - - >>> product2 = Product(product) - >>> print(product2) - Product(name='Desktop PC', price=1000) - - >>> product3 = product2.copy() - >>> print(product3) - Product(name='Desktop PC', price=1000) - Creating dicts from items:: >>> dict(product) # create a dict from all populated values diff --git a/scrapy/item.py b/scrapy/item.py index aa05e9c69..031b80a2d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst from pprint import pformat from collections import MutableMapping +from copy import deepcopy from abc import ABCMeta import six @@ -96,6 +97,13 @@ class DictItem(MutableMapping, BaseItem): def copy(self): return self.__class__(self) + def deepcopy(self): + """Return a `deep copy`_ of this item. + + .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy + """ + return deepcopy(self) + @six.add_metaclass(ItemMeta) class Item(DictItem): diff --git a/tests/test_item.py b/tests/test_item.py index 2c1eb0dd3..010d3b141 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -249,6 +249,14 @@ class ItemTest(unittest.TestCase): copied_item['name'] = copied_item['name'].upper() self.assertNotEqual(item['name'], copied_item['name']) + def test_deepcopy(self): + class TestItem(Item): + tags = Field() + item = TestItem({'tags': ['tag1']}) + copied_item = item.deepcopy() + item['tags'].append('tag2') + assert item['tags'] != copied_item['tags'] + class ItemMetaTest(unittest.TestCase): From 9a0fe8bf2dc108b1f2c50aaef211b39981e65c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez=20Cabot?= Date: Wed, 20 Mar 2019 16:13:31 +0100 Subject: [PATCH 0361/1853] remove duplicated entry in gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7392ed31e..ff6e2ea65 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ htmlcov/ .pytest_cache/ .coverage.* .cache/ -.pytest_cache/ # Windows Thumbs.db From bbf24b7a1ce2e91eab57d1b8524d398822a1ddd1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 18:02:31 -0300 Subject: [PATCH 0362/1853] Rule.process_request: use scrapy.utils.python.get_func_args --- scrapy/spiders/crawl.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index c01f75798..f474b0a18 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -10,6 +10,7 @@ import six from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.python import get_func_args from scrapy.spiders import Spider @@ -35,10 +36,8 @@ class Rule(object): Wrapper around the request processing function to maintain backward compatibility with functions that do not take a Response object as parameter. """ - argcount = self.process_request.__code__.co_argcount - if hasattr(self.process_request, '__self__'): - argcount = argcount - 1 - args = [request] if argcount == 1 else [request, response] + arg_count = len(get_func_args(self.process_request)) + args = [request] if arg_count == 1 else [request, response] return self.process_request(*args) From 56929e77d98391255b77ffd3350abb49da18009e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 18:34:55 -0300 Subject: [PATCH 0363/1853] Rule.process_request: deprecate the use of functions taking only one argument --- scrapy/spiders/crawl.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f474b0a18..f469891d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst """ import copy +import warnings + import six +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_func_args from scrapy.spiders import Spider -def identity(x): - return x +def _identity(request, response): + return request class Rule(object): @@ -25,19 +28,21 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request = process_request or identity - if follow is None: - self.follow = False if callback else True - else: - self.follow = follow + self.process_request = process_request or _identity + self.follow = follow if follow is not None else not callback def _process_request(self, request, response): """ - Wrapper around the request processing function to maintain backward compatibility - with functions that do not take a Response object as parameter. + Wrapper around the request processing function to maintain backward + compatibility with functions that do not take a Response object """ arg_count = len(get_func_args(self.process_request)) - args = [request] if arg_count == 1 else [request, response] + if arg_count == 1: + args = [request] + msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' + warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + else: + args = [request, response] return self.process_request(*args) From 174ba3cc5671cdc9e66cb29275986ff7481affc5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 19:16:18 -0300 Subject: [PATCH 0364/1853] Rule.process_request: update docs --- docs/topics/spiders.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 24b6f7ec9..30e15906e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -403,11 +403,11 @@ Crawling rules This is mainly used for filtering purposes. ``process_request`` is a callable (or a string, in which case a method from - the spider object with that name will be used) which will be called for - every request extracted by this rule. This callable should take a Request object - as first positional argument and, optionally, the Response object from which the - Request originated as second positional argument. It must return a request or None - (to filter out the request). + the spider object with that name will be used) which will be called for every + :class:`~scrapy.http.Request` extracted by this rule. This callable should + take said request as first argument and the :class:`~scrapy.http.Response` + from which the request originated as second argument. It must return a + ``Request`` object or ``None`` (to filter out the request). CrawlSpider example ~~~~~~~~~~~~~~~~~~~ From 1b4385b7e3f78694c0378455644b539d80d293a2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 19:46:17 -0300 Subject: [PATCH 0365/1853] Rule.process_request: move deprecation warnings and compiling code, update tests --- scrapy/spiders/crawl.py | 35 +++++++++++++++++++---------------- tests/test_spider.py | 38 ++++++++++++++++++++++---------------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f469891d0..6db3a1e06 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -21,6 +21,13 @@ def _identity(request, response): return request +def _get_method(method, spider): + if callable(method): + return method + elif isinstance(method, six.string_types): + return getattr(spider, method, None) + + class Rule(object): def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): @@ -29,20 +36,24 @@ class Rule(object): self.cb_kwargs = cb_kwargs or {} self.process_links = process_links self.process_request = process_request or _identity + self.process_request_argcount = None self.follow = follow if follow is not None else not callback + def _compile(self, spider): + self.callback = _get_method(self.callback, spider) + self.process_links = _get_method(self.process_links, spider) + self.process_request = _get_method(self.process_request, spider) + self.process_request_argcount = len(get_func_args(self.process_request)) + if self.process_request_argcount == 1: + msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' + warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + def _process_request(self, request, response): """ Wrapper around the request processing function to maintain backward compatibility with functions that do not take a Response object """ - arg_count = len(get_func_args(self.process_request)) - if arg_count == 1: - args = [request] - msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' - warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) - else: - args = [request, response] + args = [request] if self.process_request_argcount == 1 else [request, response] return self.process_request(*args) @@ -98,17 +109,9 @@ class CrawlSpider(Spider): yield request_or_item def _compile_rules(self): - def get_method(method): - if callable(method): - return method - elif isinstance(method, six.string_types): - return getattr(self, method, None) - self._rules = [copy.copy(r) for r in self.rules] for rule in self._rules: - rule.callback = get_method(rule.callback) - rule.process_links = get_method(rule.process_links) - rule.process_request = get_method(rule.process_request) + rule._compile(self) @classmethod def from_crawler(cls, crawler, *args, **kwargs): diff --git a/tests/test_spider.py b/tests/test_spider.py index c9af7a2d7..83fb68c2f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -275,14 +275,17 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request=process_request_change_domain), ) - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.com/somepage/item/12.html', - 'http://example.com/about.html', - 'http://example.com/nofollow.html']) + with warnings.catch_warnings(record=True) as cw: + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) + self.assertEqual(len(cw), 1) + self.assertEqual(cw[0].category, ScrapyDeprecationWarning) def test_process_request_with_response(self): @@ -324,14 +327,17 @@ class CrawlSpiderTest(SpiderTest): def process_request_upper(self, request): return request.replace(url=request.url.upper()) - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + with warnings.catch_warnings(record=True) as cw: + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + self.assertEqual(len(cw), 1) + self.assertEqual(cw[0].category, ScrapyDeprecationWarning) def test_process_request_instance_method_with_response(self): From 90934959d07db881aec5933fd7b77bcd2dccfa4f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 27 Dec 2018 17:12:24 +0500 Subject: [PATCH 0366/1853] actually apply __slots__ suggestion [wip] refactoring * SlotPriorityQueues doesn't care about objects inside, it is now just a container for multiple priority queues * assorted variable renames * don't inherit DownloaderAwarePriorityQueue from SlotBasedPriorityQueue * apply @whalebot-helmsman's suggestions for __slots__ and meta issues more bike-shedding * remove mutable default arguments * more verbose variable names remove unneeded code * PriorityAsTupleQueue.is_empty does the same as len(self) == 0 * custom PriorityAsTupleQueue.close is not needed after a switch to namedtuples * is_new and is_empty return values are unused * "url" local variable is unused PrioritySlot.__str__ shouldn't return unicode in Python 2 also, do some bike-shedding: _pathable -> _path_safe use namedtuple for PrioritySlot cleanup: _get_from_request does the same here Request.meta is always a dict --- scrapy/pqueues.py | 180 ++++++++++++++++++---------------------- tests/test_scheduler.py | 2 +- 2 files changed, 82 insertions(+), 100 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index d9effc9d1..3ef896b99 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,6 +1,6 @@ import hashlib import logging -from six import text_type +from collections import namedtuple from six.moves.urllib.parse import urlparse from queuelib import PriorityQueue @@ -17,12 +17,12 @@ logger = logging.getLogger(__name__) SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT -def _get_from_request(request, key, default=None): +def _get_request_meta(request): if isinstance(request, dict): - return request.get(key, default) + return request.setdefault('meta', {}) if isinstance(request, Request): - return getattr(request, key, default) + return request.meta raise ValueError('Bad type of request "%s"' % (request.__class__, )) @@ -35,15 +35,8 @@ def _scheduler_slot_write(request, slot): request.meta[SCHEDULER_SLOT_META_KEY] = slot -def _scheduler_slot(request): - - if isinstance(request, dict): - meta = request.get('meta', dict()) - elif isinstance(request, Request): - meta = request.meta - else: - raise ValueError('Bad type of request "%s"' % (request.__class__, )) - +def _set_scheduler_slot(request): + meta = _get_request_meta(request) slot = meta.get(SCHEDULER_SLOT_META_KEY, None) if slot is not None: @@ -53,43 +46,29 @@ def _scheduler_slot(request): url = request.get('url', None) slot = urlparse(url).hostname or '' elif isinstance(request, Request): - url = request.url slot = urlparse_cached(request).hostname or '' meta[SCHEDULER_SLOT_META_KEY] = slot - return slot -def _pathable(x): - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x]) - +def _path_safe(text): + """ Return a filesystem-safe version of a string ``text`` """ + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' + for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(x.encode('utf8')).hexdigest() - + unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() return '-'.join([pathable_slot, unique_slot]) -class PrioritySlot: - __slots__ = ('priority', 'slot') - - def __init__(self, priority=0, slot=None): - self.priority = priority - self.slot = slot - - def __hash__(self): - return hash((self.priority, self.slot)) - - def __eq__(self, other): - return (self.priority, self.slot) == (other.priority, other.slot) - - def __lt__(self, other): - return (self.priority, self.slot) < (other.priority, other.slot) +class PrioritySlot(namedtuple("PrioritySlot", ["priority", "slot"])): + """ ``(priority, slot)`` tuple which uses a path-safe slot name + when converting to str """ + __slots__ = () def __str__(self): - return '_'.join([text_type(self.priority), - _pathable(text_type(self.slot))]) + return '%s_%s' % (self.priority, _path_safe(str(self.slot))) class PriorityAsTupleQueue(PriorityQueue): @@ -99,78 +78,65 @@ class PriorityAsTupleQueue(PriorityQueue): json serializable structures """ def __init__(self, qfactory, startprios=()): - + startprios = [PrioritySlot(priority=p[0], slot=p[1]) + for p in startprios] super(PriorityAsTupleQueue, self).__init__( - qfactory, - [PrioritySlot(priority=p[0], slot=p[1]) for p in startprios] - ) - - def close(self): - startprios = super(PriorityAsTupleQueue, self).close() - return [(s.priority, s.slot) for s in startprios] - - def is_empty(self): - return not self.queues or len(self) == 0 + qfactory=qfactory, + startprios=startprios) -class SlotBasedPriorityQueue(object): +class SlotPriorityQueues(object): + """ Container for multiple priority queues. """ + def __init__(self, pqfactory, slot_startprios=None): + """ + ``pqfactory`` is a factory for creating new PriorityQueues. + It must be a function which accepts a single optional ``startprios`` + argument, with a list of priorities to create queues for. - def __init__(self, qfactory, startprios={}): - self.pqueues = dict() # slot -> priority queue - self.qfactory = qfactory # factory for creating new internal queues - - if not startprios: - return - - if not isinstance(startprios, dict): - raise ValueError("Looks like your priorities file malforfemed. " - "Possible reason: You run scrapy with previous " - "version. Interrupted it. Updated scrapy. And " - "run again.") - - for slot, prios in startprios.items(): - self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory, prios) + ``slot_startprios`` is a ``{slot: startprios}`` dict. + """ + self.pqfactory = pqfactory + self.pqueues = {} # slot -> priority queue + for slot, startprios in (slot_startprios or {}).items(): + self.pqueues[slot] = self.pqfactory(startprios) def pop_slot(self, slot): + """ Pop an object from a priority queue for this slot """ queue = self.pqueues[slot] request = queue.pop() - is_empty = queue.is_empty() - if is_empty: + if len(queue) == 0: del self.pqueues[slot] + return request - return request, is_empty - - def push_slot(self, request, priority): - slot = _scheduler_slot(request) - is_new = False + def push_slot(self, slot, obj, priority): + """ Push an object to a priority queue for this slot """ if slot not in self.pqueues: - self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory) + self.pqueues[slot] = self.pqfactory() queue = self.pqueues[slot] - is_new = queue.is_empty() - queue.push(request, PrioritySlot(priority=priority, slot=slot)) - return slot, is_new + queue.push(obj, priority) def close(self): - startprios = dict() - for slot, queue in self.pqueues.items(): - prios = queue.close() - startprios[slot] = prios + active = {slot: queue.close() + for slot, queue in self.pqueues.items()} self.pqueues.clear() - return startprios + return active def __len__(self): return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + def __contains__(self, slot): + return slot in self.pqueues -class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): + +class DownloaderAwarePriorityQueue(object): _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' @classmethod - def from_crawler(cls, crawler, qfactory, startprios={}): + def from_crawler(cls, crawler, qfactory, startprios=None): return cls(crawler, qfactory, startprios) - def __init__(self, crawler, qfactory, startprios={}): + def __init__(self, crawler, qfactory, startprios=None): ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) @@ -179,16 +145,25 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): ip_concurrency_key, ip_concurrency)) - super(DownloaderAwarePriorityQueue, self).__init__(qfactory, - startprios) - self._slots = {slot: 0 for slot in self.pqueues} + def pqfactory(startprios=()): + return PriorityAsTupleQueue(qfactory, startprios) + + if startprios and not isinstance(startprios, dict): + raise ValueError("DownloaderAwarePriorityQueue accepts " + "``startprios`` as a dict; %r instance is passed." + " Only a crawl started with the same priority " + "queue class can be resumed." % startprios.__class__) + self._slot_pqueues = SlotPriorityQueues(pqfactory, + slot_startprios=startprios) + + self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, signal=response_downloaded) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) def mark(self, request): - meta = _get_from_request(request, 'meta', None) + meta = _get_request_meta(request) if not isinstance(meta, dict): raise ValueError('No meta attribute in %s' % (request, )) meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) @@ -197,39 +172,46 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) def pop(self): - slots = [(d, s) for s, d in self._slots.items() if s in self.pqueues] + slots = [(active_downloads, slot) + for slot, active_downloads in self._active_downloads.items() + if slot in self._slot_pqueues] if not slots: return slot = min(slots)[1] - request, _ = self.pop_slot(slot) + request = self._slot_pqueues.pop_slot(slot) self.mark(request) return request def push(self, request, priority): - slot, _ = self.push_slot(request, priority) - if slot not in self._slots: - self._slots[slot] = 0 + slot = _set_scheduler_slot(request) + priority_slot = PrioritySlot(priority=priority, slot=slot) + self._slot_pqueues.push_slot(slot, request, priority_slot) + if slot not in self._active_downloads: + self._active_downloads[slot] = 0 def on_response_download(self, response, request, spider): if not self.check_mark(request): return slot = _scheduler_slot_read(request) - if slot not in self._slots or self._slots[slot] <= 0: + if slot not in self._active_downloads or self._active_downloads[slot] <= 0: raise ValueError('Get response for wrong slot "%s"' % (slot, )) - self._slots[slot] = self._slots[slot] - 1 - if self._slots[slot] == 0 and slot not in self.pqueues: - del self._slots[slot] + self._active_downloads[slot] = self._active_downloads[slot] - 1 + if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues: + del self._active_downloads[slot] def on_request_reached_downloader(self, request, spider): if not self.check_mark(request): return slot = _scheduler_slot_read(request) - self._slots[slot] = self._slots.get(slot, 0) + 1 + self._active_downloads[slot] = self._active_downloads.get(slot, 0) + 1 def close(self): - self._slots.clear() - return super(DownloaderAwarePriorityQueue, self).close() + self._active_downloads.clear() + return self._slot_pqueues.close() + + def __len__(self): + return len(self._slot_pqueues) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 17b706bd7..5dd35f45c 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -260,7 +260,7 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, ) requests.append(request) - self.assertEqual(self.scheduler.mqs._slots, {}) + self.assertEqual(self.scheduler.mqs._active_downloads, {}) self.assertEqual(len(slots), len(_SLOTS)) for request in requests: From 757f53a32461ef0c3d2fe4caf64197f67271b5f3 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 9 Jan 2019 10:00:13 +0000 Subject: [PATCH 0367/1853] Address Lucy's comments add tests to check correctness of slot setermination unmark requests after downloading shorter better exception message --- scrapy/pqueues.py | 15 ++++++++++++--- tests/test_scheduler.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 3ef896b99..d8eed010f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -36,6 +36,12 @@ def _scheduler_slot_write(request, slot): def _set_scheduler_slot(request): + """ + >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://bar.com'}) + False + >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://foo.com'}) + True + """ meta = _get_request_meta(request) slot = meta.get(SCHEDULER_SLOT_META_KEY, None) @@ -141,9 +147,8 @@ class DownloaderAwarePriorityQueue(object): ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) if ip_concurrency > 0: - raise ValueError('"%s" does not support %s=%d' % (self.__class__, - ip_concurrency_key, - ip_concurrency)) + raise ValueError('"%s" does not support setting %s' % (self.__class__, + ip_concurrency_key)) def pqfactory(startprios=()): return PriorityAsTupleQueue(qfactory, startprios) @@ -171,6 +176,9 @@ class DownloaderAwarePriorityQueue(object): def check_mark(self, request): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) + def unmark(self, request): + del request.meta[self._DOWNLOADER_AWARE_PQ_ID] + def pop(self): slots = [(active_downloads, slot) for slot, active_downloads in self._active_downloads.items() @@ -194,6 +202,7 @@ class DownloaderAwarePriorityQueue(object): def on_response_download(self, response, request, spider): if not self.check_mark(request): return + self.unmark(request) slot = _scheduler_slot_read(request) if slot not in self._active_downloads or self._active_downloads[slot] <= 0: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 5dd35f45c..3fb70a110 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -248,8 +248,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, self.close_scheduler() self.create_scheduler() - slots = list() - requests = list() + slots = [] + requests = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() slots.append(_scheduler_slot_read(request)) From 3b1db71dac8716878ff1b94ee0d1095e5c80795f Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 9 Jan 2019 12:14:40 +0000 Subject: [PATCH 0368/1853] New signal update signature documentation for new signal utilize new signal correct signal handler signature emit new signal test another signal new signal rename test file faster test rename test case tests for signal emitting in bad cases --- docs/topics/signals.rst | 17 +++++++++ scrapy/core/downloader/__init__.py | 3 ++ scrapy/pqueues.py | 6 +-- scrapy/signals.py | 1 + tests/test_request_left.py | 59 ++++++++++++++++++++++++++++++ tests/test_scheduler.py | 8 ++-- 6 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 tests/test_request_left.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ff07b9d55..f13e8270c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,6 +295,23 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_left_downloader +--------------------------- + +.. signal:: request_left_downloader +.. function:: request_left_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request` left downloader even in case of + failure. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4695d75f4..d856a2f37 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -188,6 +188,9 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) + self.signals.send_catch_log(signal=signals.request_left_downloader, + request=request, + spider=spider) return _ return dfd.addBoth(finish_transferring) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index d8eed010f..6a9feb599 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -7,7 +7,7 @@ from queuelib import PriorityQueue from scrapy.core.downloader import Downloader from scrapy.http import Request -from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.utils.httpobj import urlparse_cached @@ -163,7 +163,7 @@ class DownloaderAwarePriorityQueue(object): self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, - signal=response_downloaded) + signal=request_left_downloader) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) @@ -199,7 +199,7 @@ class DownloaderAwarePriorityQueue(object): if slot not in self._active_downloads: self._active_downloads[slot] = 0 - def on_response_download(self, response, request, spider): + def on_response_download(self, request, spider): if not self.check_mark(request): return self.unmark(request) diff --git a/scrapy/signals.py b/scrapy/signals.py index c0e4bb74e..2ea986b8c 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,6 +14,7 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() +request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() diff --git a/tests/test_request_left.py b/tests/test_request_left.py new file mode 100644 index 000000000..ddeca0499 --- /dev/null +++ b/tests/test_request_left.py @@ -0,0 +1,59 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.signals import request_left_downloader +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + +class SignalCatcherSpider(Spider): + name = 'signal_catcher' + + def __init__(self, crawler, url, *args, **kwargs): + super(SignalCatcherSpider, self).__init__(*args, **kwargs) + crawler.signals.connect(self.on_response_download, + signal=request_left_downloader) + self.catched_times = 0 + self.start_urls = [url] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(crawler, *args, **kwargs) + return spider + + def on_response_download(self, request, spider): + self.catched_times = self.catched_times + 1 + + +class TestCatching(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_success(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/status?n=200")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_timeout(self): + crawler = get_crawler(SignalCatcherSpider, + {'DOWNLOAD_TIMEOUT': 0.1}) + yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_disconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/drop")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_noconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') + self.assertEqual(crawler.spider.catched_times, 1) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 3fb70a110..1bcc1e5a8 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -9,7 +9,7 @@ from scrapy.crawler import Crawler from scrapy.core.scheduler import Scheduler from scrapy.http import Request from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write -from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -216,9 +216,8 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, for request in requests: self.mock_crawler.signals.send_catch_log( - signal=response_downloaded, + signal=request_left_downloader, request=request, - response=None, spider=self.spider ) @@ -265,9 +264,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, for request in requests: self.mock_crawler.signals.send_catch_log( - signal=response_downloaded, + signal=request_left_downloader, request=request, - response=None, spider=self.spider ) From 83eb5376458ce1d444e8ad7911730ee0c58c8544 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 17 Jan 2019 07:38:15 +0500 Subject: [PATCH 0369/1853] assorted cleanups: comments, docstrings, etc scheduler cleanup Scheduler no longer converts requests to dicts; PriorityQueue instances always work with Request instances; converting Requests to dicts is now Priority Queue responsibility. minor cleanup --- docs/topics/settings.rst | 6 +- scrapy/core/scheduler.py | 99 +++++++++++++---- scrapy/pqueues.py | 158 +++++++++++++++------------- scrapy/settings/default_settings.py | 2 +- scrapy/squeues.py | 11 +- 5 files changed, 175 insertions(+), 101 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7b9ff7e39..6e13e64d6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1142,13 +1142,13 @@ Type of in-memory queue used by scheduler. Other available type is: SCHEDULER_PRIORITY_QUEUE ------------------------ -Default: ``'queuelib.PriorityQueue'`` +Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. ``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than -``'queuelib.PriorityQueue'`` when you crawl many different domains in parallel. -But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different +domains in parallel. But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. .. setting:: SPIDER_CONTRACTS diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d40f3aa0c..c385fafe1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,17 +1,44 @@ import os import json import logging +import warnings from os.path import join, exists -from scrapy.utils.reqser import request_to_dict, request_from_dict +from queuelib import PriorityQueue + from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir +from scrapy.utils.deprecate import ScrapyDeprecationWarning + logger = logging.getLogger(__name__) class Scheduler(object): + """ + Scrapy Scheduler. It allows to enqueue requests and then get + a next request to download. Scheduler is also handling duplication + filtering, via dupefilter. + Prioritization and queueing is not performed by the Scheduler. + User sets ``priority`` field for each Request, and a PriorityQueue + (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities + to dequeue requests in a desired order. + + Scheduler uses two PriorityQueue instances, configured to work in-memory + and on-disk (optional). When on-disk queue is present, it is used by + default, and an in-memory queue is used as a fallback for cases where + a disk queue can't handle a request (can't serialize it). + + :setting:`SCHEDULER_MEMORY_QUEUE` and + :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes + which PriorityQueue instances would be instantiated with, to keep requests + on disk and in memory respectively. + + Overall, Scheduler is an object which holds several PriorityQueue instances + (in-memory and on-disk) and implements fallback logic for them. + Also, it handles dupefilters. + """ def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None, pqclass=None, crawler=None): self.df = dupefilter @@ -29,9 +56,19 @@ class Scheduler(object): dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) + if pqclass is PriorityQueue: + # backwards compatibility + warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" + " is no longer supported because of API changes; " + "please use 'scrapy.pqueues.ScrapyPriorityQueue'", + ScrapyDeprecationWarning) + from scrapy.pqueues import ScrapyPriorityQueue + pqclass = ScrapyPriorityQueue + dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG')) + logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', + settings.getbool('SCHEDULER_DEBUG')) return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass, crawler=crawler) @@ -41,15 +78,19 @@ class Scheduler(object): def open(self, spider): self.spider = spider - self.mqs = create_instance(self.pqclass, None, self.crawler, self._newmq) + + # in-memory PriorityQueue instance + self.mqs = self._mq() + + # on-disk PriorityQueue instance self.dqs = self._dq() if self.dqdir else None + return self.df.open() def close(self, reason): if self.dqs: - prios = self.dqs.close() - with open(join(self.dqdir, 'active.json'), 'w') as f: - json.dump(prios, f) + state = self.dqs.close() + self._write_dqs_state(self.dqdir, state) return self.df.close(reason) def enqueue_request(self, request): @@ -66,7 +107,7 @@ class Scheduler(object): return True def next_request(self): - request = self.mqs.pop() + request = self._mqpop() if request: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: @@ -84,8 +125,7 @@ class Scheduler(object): if self.dqs is None: return try: - reqd = request_to_dict(request, self.spider) - self.dqs.push(reqd, -request.priority) + self.dqs.push(request, -request.priority) except ValueError as e: # non serializable request if self.logunser: msg = ("Unable to serialize request: %(request)s - reason:" @@ -105,37 +145,54 @@ class Scheduler(object): def _dqpop(self): if self.dqs: - d = self.dqs.pop() - if d: - return request_from_dict(d, self.spider) + return self.dqs.pop() + + def _mqpop(self): + return self.mqs.pop() def _newmq(self, priority): + """ Factory for creating memory queues. """ return self.mqclass() def _newdq(self, priority): - return self.dqclass(join(self.dqdir, 'p%s' % (priority, ))) + """ Factory for creating disk queues. """ + path = join(self.dqdir, 'p%s' % (priority, )) + return self.dqclass(path) + + def _mq(self): + """ Create a new priority queue instance, with in-memory storage """ + return create_instance(self.pqclass, None, self.crawler, self._newmq, + serialize=False) def _dq(self): - activef = join(self.dqdir, 'active.json') - if exists(activef): - with open(activef) as f: - prios = json.load(f) - else: - prios = () - + """ Create a new priority queue instance, with disk storage """ + state = self._read_dqs_state(self.dqdir) q = create_instance(self.pqclass, None, self.crawler, self._newdq, - startprios=prios) + state, + serialize=True) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", {'queuesize': len(q)}, extra={'spider': self.spider}) return q def _dqdir(self, jobdir): + """ Return a folder name to keep disk queue state at """ if jobdir: dqdir = join(jobdir, 'requests.queue') if not exists(dqdir): os.makedirs(dqdir) return dqdir + + def _read_dqs_state(self, dqdir): + path = join(dqdir, 'active.json') + if not exists(path): + return () + with open(path) as f: + return json.load(f) + + def _write_dqs_state(self, dqdir, state): + with open(join(dqdir, 'active.json'), 'w') as f: + json.dump(state, f) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 6a9feb599..622f6bbc5 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,10 +1,10 @@ import hashlib import logging from collections import namedtuple -from six.moves.urllib.parse import urlparse from queuelib import PriorityQueue +from scrapy.utils.reqser import request_to_dict, request_from_dict from scrapy.core.downloader import Downloader from scrapy.http import Request from scrapy.signals import request_reached_downloader, request_left_downloader @@ -17,16 +17,6 @@ logger = logging.getLogger(__name__) SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT -def _get_request_meta(request): - if isinstance(request, dict): - return request.setdefault('meta', {}) - - if isinstance(request, Request): - return request.meta - - raise ValueError('Bad type of request "%s"' % (request.__class__, )) - - def _scheduler_slot_read(request, default=None): return request.meta.get(SCHEDULER_SLOT_META_KEY, default) @@ -37,24 +27,17 @@ def _scheduler_slot_write(request, slot): def _set_scheduler_slot(request): """ - >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://bar.com'}) - False - >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://foo.com'}) - True + >>> request = Request('http://example.com') + >>> _set_scheduler_slot(request) + 'example.com' + >>> _scheduler_slot_read(request) + 'example.com' """ - meta = _get_request_meta(request) - slot = meta.get(SCHEDULER_SLOT_META_KEY, None) - + slot = _scheduler_slot_read(request, None) if slot is not None: return slot - - if isinstance(request, dict): - url = request.get('url', None) - slot = urlparse(url).hostname or '' - elif isinstance(request, Request): - slot = urlparse_cached(request).hostname or '' - - meta[SCHEDULER_SLOT_META_KEY] = slot + slot = urlparse_cached(request).hostname or '' + _scheduler_slot_write(request, slot) return slot @@ -68,30 +51,25 @@ def _path_safe(text): return '-'.join([pathable_slot, unique_slot]) -class PrioritySlot(namedtuple("PrioritySlot", ["priority", "slot"])): - """ ``(priority, slot)`` tuple which uses a path-safe slot name - when converting to str """ +class _Priority(namedtuple("_Priority", ["priority", "slot"])): + """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple + which can be used instead of int priorities in queues: + + * they are ordered in the same way - order is still by priority value, + min(prios) works; + * str(p) representation is guaranteed to be different when slots + are different - this is important because str(p) is used to create + queue files on disk; + * they have readable str(p) representation which is safe + to use as a file name. + """ __slots__ = () def __str__(self): return '%s_%s' % (self.priority, _path_safe(str(self.slot))) -class PriorityAsTupleQueue(PriorityQueue): - """ - Python structures is not directly (de)serialized (to)from json. - We need this modified queue to transform custom structure (from)to - json serializable structures - """ - def __init__(self, qfactory, startprios=()): - startprios = [PrioritySlot(priority=p[0], slot=p[1]) - for p in startprios] - super(PriorityAsTupleQueue, self).__init__( - qfactory=qfactory, - startprios=startprios) - - -class SlotPriorityQueues(object): +class _SlotPriorityQueues(object): """ Container for multiple priority queues. """ def __init__(self, pqfactory, slot_startprios=None): """ @@ -134,44 +112,78 @@ class SlotPriorityQueues(object): return slot in self.pqueues -class DownloaderAwarePriorityQueue(object): - - _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' +class ScrapyPriorityQueue(PriorityQueue): + """ + PriorityQueue which works with scrapy.Request instances and + can optionally convert them to/from dicts before/after putting to a queue. + """ + def __init__(self, crawler, qfactory, startprios=(), serialize=False): + super(ScrapyPriorityQueue, self).__init__(qfactory, startprios) + self.serialize = serialize + self.spider = crawler.spider @classmethod - def from_crawler(cls, crawler, qfactory, startprios=None): - return cls(crawler, qfactory, startprios) + def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False): + return cls(crawler, qfactory, startprios, serialize) - def __init__(self, crawler, qfactory, startprios=None): - ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' - ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) + def push(self, request, priority=0): + if self.serialize: + request = request_to_dict(request, self.spider) + super(ScrapyPriorityQueue, self).push(request, priority) - if ip_concurrency > 0: - raise ValueError('"%s" does not support setting %s' % (self.__class__, - ip_concurrency_key)) + def pop(self): + request = super(ScrapyPriorityQueue, self).pop() + if request and self.serialize: + request = request_from_dict(request, self.spider) + return request + + +class DownloaderAwarePriorityQueue(object): + """ PriorityQueue which takes Downlaoder activity in account: + domains (slots) with the least amount of active downloads are dequeued + first. + """ + _DOWNLOADER_AWARE_PQ_ID = '_DOWNLOADER_AWARE_PQ_ID' + + @classmethod + def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False): + return cls(crawler, qfactory, slot_startprios, serialize) + + def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False): + if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: + raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' + % (self.__class__,)) + + if slot_startprios and not isinstance(slot_startprios, dict): + raise ValueError("DownloaderAwarePriorityQueue accepts " + "``slot_startprios`` as a dict; %r instance " + "is passed. Most likely, it means the state is" + "created by an incompatible priority queue. " + "Only a crawl started with the same priority " + "queue class can be resumed." % + slot_startprios.__class__) + + slot_startprios = { + slot: [_Priority(p, slot) for p in startprios] + for slot, startprios in (slot_startprios or {}).items()} def pqfactory(startprios=()): - return PriorityAsTupleQueue(qfactory, startprios) - - if startprios and not isinstance(startprios, dict): - raise ValueError("DownloaderAwarePriorityQueue accepts " - "``startprios`` as a dict; %r instance is passed." - " Only a crawl started with the same priority " - "queue class can be resumed." % startprios.__class__) - self._slot_pqueues = SlotPriorityQueues(pqfactory, - slot_startprios=startprios) + return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize) + self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios) self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, signal=request_left_downloader) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) + self.serialize = serialize + # There are two PriorityQueues at the same time (memory and disk-based), + # and they both listen to Downloader signals. To filter out signals + # coming from the other queue, each queue keeps track of its own + # requests using mark / unmark / check_mark methods. def mark(self, request): - meta = _get_request_meta(request) - if not isinstance(meta, dict): - raise ValueError('No meta attribute in %s' % (request, )) - meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) + request.meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) def check_mark(self, request): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) @@ -194,7 +206,7 @@ class DownloaderAwarePriorityQueue(object): def push(self, request, priority): slot = _set_scheduler_slot(request) - priority_slot = PrioritySlot(priority=priority, slot=slot) + priority_slot = _Priority(priority=priority, slot=slot) self._slot_pqueues.push_slot(slot, request, priority_slot) if slot not in self._active_downloads: self._active_downloads[slot] = 0 @@ -206,8 +218,8 @@ class DownloaderAwarePriorityQueue(object): slot = _scheduler_slot_read(request) if slot not in self._active_downloads or self._active_downloads[slot] <= 0: - raise ValueError('Get response for wrong slot "%s"' % (slot, )) - self._active_downloads[slot] = self._active_downloads[slot] - 1 + raise ValueError('Got response for a wrong slot "%s"' % (slot, )) + self._active_downloads[slot] -= 1 if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues: del self._active_downloads[slot] @@ -220,7 +232,9 @@ class DownloaderAwarePriorityQueue(object): def close(self): self._active_downloads.clear() - return self._slot_pqueues.close() + active = self._slot_pqueues.close() + return {slot: [p.priority for p in startprios] + for slot, startprios in active.items()} def __len__(self): return len(self._slot_pqueues) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..365b405cb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -244,7 +244,7 @@ ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue' SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' -SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue' +SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' SPIDER_LOADER_WARN_ONLY = False diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d2074a457..30cc926e5 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,6 +7,7 @@ from six.moves import cPickle as pickle from queuelib import queue + def _serializable_queue(queue_class, serialize, deserialize): class SerializableQueue(queue_class): @@ -22,6 +23,7 @@ def _serializable_queue(queue_class, serialize, deserialize): return SerializableQueue + def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) @@ -31,13 +33,14 @@ def _pickle_serialize(obj): except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) -PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ + +PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, _pickle_serialize, pickle.loads) -PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \ +PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, _pickle_serialize, pickle.loads) -MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ +MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, marshal.dumps, marshal.loads) -MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \ +MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, marshal.dumps, marshal.loads) FifoMemoryQueue = queue.FifoMemoryQueue LifoMemoryQueue = queue.LifoMemoryQueue From 443fb98a4776f4196662bb48918f1471758b7ae7 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 5 Mar 2019 12:44:07 +0000 Subject: [PATCH 0370/1853] Use downloader directly rename variable remove old write function remove unused imports remove old read function remove unused function use mock methods mock downloader close downloader add parse method use new PQ class create mock downloader use downloader directly remove mark/unmark mechanism --- scrapy/pqueues.py | 103 ++++++++++------------------------------ tests/test_scheduler.py | 87 +++++++++++++++++++++------------ 2 files changed, 81 insertions(+), 109 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 622f6bbc5..0681e6729 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -5,42 +5,11 @@ from collections import namedtuple from queuelib import PriorityQueue from scrapy.utils.reqser import request_to_dict, request_from_dict -from scrapy.core.downloader import Downloader -from scrapy.http import Request -from scrapy.signals import request_reached_downloader, request_left_downloader -from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) -SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT - - -def _scheduler_slot_read(request, default=None): - return request.meta.get(SCHEDULER_SLOT_META_KEY, default) - - -def _scheduler_slot_write(request, slot): - request.meta[SCHEDULER_SLOT_META_KEY] = slot - - -def _set_scheduler_slot(request): - """ - >>> request = Request('http://example.com') - >>> _set_scheduler_slot(request) - 'example.com' - >>> _scheduler_slot_read(request) - 'example.com' - """ - slot = _scheduler_slot_read(request, None) - if slot is not None: - return slot - slot = urlparse_cached(request).hostname or '' - _scheduler_slot_write(request, slot) - return slot - - def _path_safe(text): """ Return a filesystem-safe version of a string ``text`` """ pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' @@ -138,6 +107,25 @@ class ScrapyPriorityQueue(PriorityQueue): return request +class DownloaderInterface(object): + + def __init__(self, crawler): + self.downloader = crawler.engine.downloader + + def stats(self, possible_slots): + return [(self._active_downloads(slot), slot) + for slot in possible_slots] + + def get_slot_key(self, request): + return self.downloader._get_slot_key(request, None) + + def _active_downloads(self, slot): + """ Return a number of requests in a Downloader for a given slot """ + if slot not in self.downloader.slots: + return 0 + return len(self.downloader.slots[slot].active) + + class DownloaderAwarePriorityQueue(object): """ PriorityQueue which takes Downlaoder activity in account: domains (slots) with the least amount of active downloads are dequeued @@ -170,68 +158,25 @@ class DownloaderAwarePriorityQueue(object): def pqfactory(startprios=()): return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize) self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios) - - self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} - crawler.signals.connect(self.on_response_download, - signal=request_left_downloader) - crawler.signals.connect(self.on_request_reached_downloader, - signal=request_reached_downloader) self.serialize = serialize - - # There are two PriorityQueues at the same time (memory and disk-based), - # and they both listen to Downloader signals. To filter out signals - # coming from the other queue, each queue keeps track of its own - # requests using mark / unmark / check_mark methods. - def mark(self, request): - request.meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) - - def check_mark(self, request): - return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) - - def unmark(self, request): - del request.meta[self._DOWNLOADER_AWARE_PQ_ID] + self._downloader_interface = DownloaderInterface(crawler) def pop(self): - slots = [(active_downloads, slot) - for slot, active_downloads in self._active_downloads.items() - if slot in self._slot_pqueues] + stats = self._downloader_interface.stats(self._slot_pqueues.pqueues) - if not slots: + if not stats: return - slot = min(slots)[1] + slot = min(stats)[1] request = self._slot_pqueues.pop_slot(slot) - self.mark(request) return request def push(self, request, priority): - slot = _set_scheduler_slot(request) + slot = self._downloader_interface.get_slot_key(request) priority_slot = _Priority(priority=priority, slot=slot) self._slot_pqueues.push_slot(slot, request, priority_slot) - if slot not in self._active_downloads: - self._active_downloads[slot] = 0 - - def on_response_download(self, request, spider): - if not self.check_mark(request): - return - self.unmark(request) - - slot = _scheduler_slot_read(request) - if slot not in self._active_downloads or self._active_downloads[slot] <= 0: - raise ValueError('Got response for a wrong slot "%s"' % (slot, )) - self._active_downloads[slot] -= 1 - if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues: - del self._active_downloads[slot] - - def on_request_reached_downloader(self, request, spider): - if not self.check_mark(request): - return - - slot = _scheduler_slot_read(request) - self._active_downloads[slot] = self._active_downloads.get(slot, 0) + 1 def close(self): - self._active_downloads.clear() active = self._slot_pqueues.close() return {slot: [p.priority for p in startprios] for slot, startprios in active.items()} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 1bcc1e5a8..75c0b7530 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,20 +1,50 @@ import shutil import tempfile import unittest +import collections from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.crawler import Crawler +from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler from scrapy.http import Request -from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write -from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.spiders import Spider +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +MockEngine = collections.namedtuple('MockEngine', ['downloader']) +MockSlot = collections.namedtuple('MockSlot', ['active']) + + +class MockDownloader: + def __init__(self): + self.slots = dict() + + def _set_slot_key(self, slot, request, spider): + request.meta[Downloader.DOWNLOAD_SLOT] = slot + + def _get_slot_key(self, request, spider): + if Downloader.DOWNLOAD_SLOT in request.meta: + return request.meta[Downloader.DOWNLOAD_SLOT] + + return urlparse_cached(request).hostname or '' + + def increment(self, slot_key): + slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot.active.append(1) + + def decrement(self, slot_key): + slot = self.slots.get(slot_key) + slot.active.pop() + + def close(self): + pass + + class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): @@ -27,6 +57,7 @@ class MockCrawler(Crawler): DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' ) super(MockCrawler, self).__init__(Spider, settings) + self.engine = MockEngine(downloader=MockDownloader()) class SchedulerHandler: @@ -42,6 +73,7 @@ class SchedulerHandler: def close_scheduler(self): self.scheduler.close('finished') self.mock_crawler.stop() + self.mock_crawler.engine.downloader.close() def setUp(self): self.create_scheduler() @@ -147,11 +179,11 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): - priority_queue_cls = 'queuelib.PriorityQueue' + priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): - priority_queue_cls = 'queuelib.PriorityQueue' + priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' _SLOTS = [("http://foo.com/a", 'a'), @@ -172,7 +204,7 @@ class TestMigration(unittest.TestCase): def _migration(self, tmp_dir): prev_scheduler_handler = SchedulerHandler() - prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue' + prev_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' prev_scheduler_handler.jobdir = tmp_dir prev_scheduler_handler.create_scheduler() @@ -196,30 +228,25 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): + downloader = self.mock_crawler.engine.downloader for url, slot in _SLOTS: request = Request(url) - _scheduler_slot_write(request, slot) + downloader._set_slot_key(slot, request, None) self.scheduler.enqueue_request(request) slots = list() requests = list() while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - slots.append(_scheduler_slot_read(request)) - self.mock_crawler.signals.send_catch_log( - signal=request_reached_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + slots.append(slot) + downloader.increment(slot) requests.append(request) self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log( - signal=request_left_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + self.mock_crawler.engine.downloader.decrement(slot) unique_slots = len(set(s for _, s in _SLOTS)) for i in range(0, len(_SLOTS), unique_slots): @@ -239,9 +266,11 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): + downloader = self.mock_crawler.engine.downloader + for url, slot in _SLOTS: request = Request(url) - _scheduler_slot_write(request, slot) + downloader._set_slot_key(slot, request, None) self.scheduler.enqueue_request(request) self.close_scheduler() @@ -249,27 +278,22 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, slots = [] requests = [] + downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - slots.append(_scheduler_slot_read(request)) - self.mock_crawler.signals.send_catch_log( - signal=request_reached_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + slots.append(slot) + downloader.increment(slot) requests.append(request) - self.assertEqual(self.scheduler.mqs._active_downloads, {}) self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log( - signal=request_left_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + downloader.decrement(slot) _is_slots_unique(_SLOTS, slots) + self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) class StartUrlsSpider(Spider): @@ -277,6 +301,9 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls + def parse(self, response): + pass + class TestIntegrationWithDownloaderAwareOnDisk(TestCase): def setUp(self): From 989bba6cb340fcc1ddb32e75ade567864d8b3884 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 7 Mar 2019 09:00:14 +0000 Subject: [PATCH 0371/1853] Revert "new signal" This reverts commit 646164fd7d6dd52061804d2df7424cff929bf739. remove tests Revert "emit new signal" This reverts commit fcde0c6880678957a76af6083b6248f430a00fcf. Revert "documentation for new signal" This reverts commit 8aeb9f696ece95c16499a96767a7afa3d9c4abf4. --- docs/topics/signals.rst | 17 --------- scrapy/core/downloader/__init__.py | 3 -- scrapy/signals.py | 1 - tests/test_request_left.py | 59 ------------------------------ 4 files changed, 80 deletions(-) delete mode 100644 tests/test_request_left.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f13e8270c..ff07b9d55 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,23 +295,6 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object -request_left_downloader ---------------------------- - -.. signal:: request_left_downloader -.. function:: request_left_downloader(request, spider) - - Sent when a :class:`~scrapy.http.Request` left downloader even in case of - failure. - - The signal does not support returning deferreds from their handlers. - - :param request: the request that reached downloader - :type request: :class:`~scrapy.http.Request` object - - :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object - response_received ----------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d856a2f37..4695d75f4 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -188,9 +188,6 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) - self.signals.send_catch_log(signal=signals.request_left_downloader, - request=request, - spider=spider) return _ return dfd.addBoth(finish_transferring) diff --git a/scrapy/signals.py b/scrapy/signals.py index 2ea986b8c..c0e4bb74e 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,7 +14,6 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() -request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() diff --git a/tests/test_request_left.py b/tests/test_request_left.py deleted file mode 100644 index ddeca0499..000000000 --- a/tests/test_request_left.py +++ /dev/null @@ -1,59 +0,0 @@ -from twisted.internet import defer -from twisted.trial.unittest import TestCase -from scrapy.signals import request_left_downloader -from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer - -class SignalCatcherSpider(Spider): - name = 'signal_catcher' - - def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) - crawler.signals.connect(self.on_response_download, - signal=request_left_downloader) - self.catched_times = 0 - self.start_urls = [url] - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = cls(crawler, *args, **kwargs) - return spider - - def on_response_download(self, request, spider): - self.catched_times = self.catched_times + 1 - - -class TestCatching(TestCase): - - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_success(self): - crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/status?n=200")) - self.assertEqual(crawler.spider.catched_times, 1) - - @defer.inlineCallbacks - def test_timeout(self): - crawler = get_crawler(SignalCatcherSpider, - {'DOWNLOAD_TIMEOUT': 0.1}) - yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) - self.assertEqual(crawler.spider.catched_times, 1) - - @defer.inlineCallbacks - def test_disconnect(self): - crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/drop")) - self.assertEqual(crawler.spider.catched_times, 1) - - @defer.inlineCallbacks - def test_noconnect(self): - crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') - self.assertEqual(crawler.spider.catched_times, 1) From 8afffb7234b282dd8bd28eec2e4eb8e3f86b5723 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 22 Mar 2019 09:12:23 +0000 Subject: [PATCH 0372/1853] Tests Cleanup add doctest for function no need in this variables move common assertion inside function rename variable rename variables rename function use function this is not a method of public API correct name for test Update docs/topics/settings.rst Co-Authored-By: whalebot-helmsman --- docs/topics/settings.rst | 4 +- tests/test_scheduler.py | 82 ++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6e13e64d6..cf454f4ec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1146,9 +1146,9 @@ Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. -``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than +``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different -domains in parallel. But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. .. setting:: SPIDER_CONTRACTS diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 75c0b7530..eaf748d35 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -24,9 +24,6 @@ class MockDownloader: def __init__(self): self.slots = dict() - def _set_slot_key(self, slot, request, spider): - request.meta[Downloader.DOWNLOAD_SLOT] = slot - def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: return request.meta[Downloader.DOWNLOAD_SLOT] @@ -186,12 +183,12 @@ class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' -_SLOTS = [("http://foo.com/a", 'a'), - ("http://foo.com/b", 'a'), - ("http://foo.com/c", 'b'), - ("http://foo.com/d", 'b'), - ("http://foo.com/e", 'c'), - ("http://foo.com/f", 'c')] +_URLS_WITH_SLOTS = [("http://foo.com/a", 'a'), + ("http://foo.com/b", 'a'), + ("http://foo.com/c", 'b'), + ("http://foo.com/d", 'b'), + ("http://foo.com/e", 'c'), + ("http://foo.com/f", 'c')] class TestMigration(unittest.TestCase): @@ -228,37 +225,52 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): - downloader = self.mock_crawler.engine.downloader - for url, slot in _SLOTS: + for url, slot in _URLS_WITH_SLOTS: request = Request(url) - downloader._set_slot_key(slot, request, None) + request.meta[Downloader.DOWNLOAD_SLOT] = slot self.scheduler.enqueue_request(request) - slots = list() + downloader = self.mock_crawler.engine.downloader + dequeued_slots = list() requests = list() while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() slot = downloader._get_slot_key(request, None) - slots.append(slot) + dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) - self.assertEqual(len(slots), len(_SLOTS)) for request in requests: slot = downloader._get_slot_key(request, None) self.mock_crawler.engine.downloader.decrement(slot) - unique_slots = len(set(s for _, s in _SLOTS)) - for i in range(0, len(_SLOTS), unique_slots): - part = slots[i:i + unique_slots] - self.assertEqual(len(part), len(set(part))) + self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), + dequeued_slots)) -def _is_slots_unique(base_slots, result_slots): - unique_slots = len(set(s for _, s in base_slots)) - for i in range(0, len(result_slots), unique_slots): - part = result_slots[i:i + unique_slots] - assert len(part) == len(set(part)) +def _is_scheduling_fair(enqueued_slots, dequeued_slots): + """ + We enqueued same number of requests for every slot. + Assert correct order, e.g. + + >>> enqueued = ['a', 'b', 'c'] * 2 + >>> correct = ['a', 'c', 'b', 'b', 'a', 'c'] + >>> incorrect = ['a', 'a', 'b', 'c', 'c', 'b'] + >>> _is_scheduling_fair(enqueued, correct) + True + >>> _is_scheduling_fair(enqueued, incorrect) + False + """ + if len(dequeued_slots) != len(enqueued_slots): + return False + + slots_number = len(set(enqueued_slots)) + for i in range(0, len(dequeued_slots), slots_number): + part = dequeued_slots[i:i + slots_number] + if len(part) != len(set(part)): + return False + + return True class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, @@ -266,33 +278,31 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): - downloader = self.mock_crawler.engine.downloader - for url, slot in _SLOTS: + for url, slot in _URLS_WITH_SLOTS: request = Request(url) - downloader._set_slot_key(slot, request, None) + request.meta[Downloader.DOWNLOAD_SLOT] = slot self.scheduler.enqueue_request(request) self.close_scheduler() self.create_scheduler() - slots = [] + dequeued_slots = list() requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() slot = downloader._get_slot_key(request, None) - slots.append(slot) + dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) - self.assertEqual(len(slots), len(_SLOTS)) - for request in requests: slot = downloader._get_slot_key(request, None) downloader.decrement(slot) - _is_slots_unique(_SLOTS, slots) + self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), + dequeued_slots)) self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) @@ -305,7 +315,7 @@ class StartUrlsSpider(Spider): pass -class TestIntegrationWithDownloaderAwareOnDisk(TestCase): +class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( StartUrlsSpider, @@ -322,10 +332,10 @@ class TestIntegrationWithDownloaderAwareOnDisk(TestCase): with MockServer() as mockserver: url = mockserver.url("/status?n=200", is_secure=False) - slots = [url] * 6 - yield self.crawler.crawl(slots) + start_urls = [url] * 6 + yield self.crawler.crawl(start_urls) self.assertEqual(self.crawler.stats.get_value('downloader/response_count'), - len(slots)) + len(start_urls)) class TestIncompatibility(unittest.TestCase): From df574de8cc5c58618f6075ca3afb14059a9e30ed Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Sat, 23 Mar 2019 00:54:39 +0800 Subject: [PATCH 0373/1853] improve tests and fix some lint warnings (#6) * refactor downloader-aware test cases * fix lint * add doctest for _path_safe * remove unused code * better doctest --- scrapy/pqueues.py | 12 +++++++-- tests/test_scheduler.py | 57 ++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 0681e6729..6ecd1b51a 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -11,7 +11,16 @@ logger = logging.getLogger(__name__) def _path_safe(text): - """ Return a filesystem-safe version of a string ``text`` """ + """ + Return a filesystem-safe version of a string ``text`` + + >>> _path_safe('simple.org').startswith('simple.org') + True + >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org') + True + >>> _path_safe('some@symbol?').startswith('some_symbol_') + True + """ pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) # as we replace some letters we can get collision for different slots @@ -131,7 +140,6 @@ class DownloaderAwarePriorityQueue(object): domains (slots) with the least amount of active downloads are dequeued first. """ - _DOWNLOADER_AWARE_PQ_ID = '_DOWNLOADER_AWARE_PQ_ID' @classmethod def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index eaf748d35..e0e3600e5 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader']) MockSlot = collections.namedtuple('MockSlot', ['active']) -class MockDownloader: +class MockDownloader(object): def __init__(self): self.slots = dict() @@ -57,7 +57,7 @@ class MockCrawler(Crawler): self.engine = MockEngine(downloader=MockDownloader()) -class SchedulerHandler: +class SchedulerHandler(object): priority_queue_cls = None jobdir = None @@ -220,34 +220,6 @@ class TestMigration(unittest.TestCase): self._migration(self.tmpdir) -class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, - unittest.TestCase): - priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' - - def test_logic(self): - for url, slot in _URLS_WITH_SLOTS: - request = Request(url) - request.meta[Downloader.DOWNLOAD_SLOT] = slot - self.scheduler.enqueue_request(request) - - downloader = self.mock_crawler.engine.downloader - dequeued_slots = list() - requests = list() - while self.scheduler.has_pending_requests(): - request = self.scheduler.next_request() - slot = downloader._get_slot_key(request, None) - dequeued_slots.append(slot) - downloader.increment(slot) - requests.append(request) - - for request in requests: - slot = downloader._get_slot_key(request, None) - self.mock_crawler.engine.downloader.decrement(slot) - - self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), - dequeued_slots)) - - def _is_scheduling_fair(enqueued_slots, dequeued_slots): """ We enqueued same number of requests for every slot. @@ -273,31 +245,33 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, - unittest.TestCase): +class DownloaderAwareSchedulerTestMixin(object): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + reopen = False def test_logic(self): - for url, slot in _URLS_WITH_SLOTS: request = Request(url) request.meta[Downloader.DOWNLOAD_SLOT] = slot self.scheduler.enqueue_request(request) - self.close_scheduler() - self.create_scheduler() + if self.reopen: + self.close_scheduler() + self.create_scheduler() dequeued_slots = list() requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() + # pylint: disable=protected-access slot = downloader._get_slot_key(request, None) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: + # pylint: disable=protected-access slot = downloader._get_slot_key(request, None) downloader.decrement(slot) @@ -306,10 +280,23 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) +class TestSchedulerWithDownloaderAwareInMemory(DownloaderAwareSchedulerTestMixin, + BaseSchedulerInMemoryTester, + unittest.TestCase): + pass + + +class TestSchedulerWithDownloaderAwareOnDisk(DownloaderAwareSchedulerTestMixin, + BaseSchedulerOnDiskTester, + unittest.TestCase): + reopen = True + + class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls + super(StartUrlsSpider, self).__init__(start_urls) def parse(self, response): pass From 31b8a6b33aed9e77a4d37a5c83b1545202207cad Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 25 Mar 2019 08:53:15 +0000 Subject: [PATCH 0374/1853] report warnings --- tests/test_crawler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 268948a70..d9ec9ee8d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import tempfile import warnings from twisted.internet import defer @@ -37,7 +36,11 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertIsInstance(spiders, sl_cls) self.crawler.spiders - self.assertEqual(len(w), 1, "Warn deprecated access only once") + is_one_warning = len(w) == 1 + if not is_one_warning: + for warning in w: + print(warning) + self.assertTrue(is_one_warning, "Warn deprecated access only once") def test_populate_spidercls_settings(self): spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} From 73e4ff5304d273404a147d06726a8ae8cae1c925 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 25 Mar 2019 13:48:58 +0000 Subject: [PATCH 0375/1853] report warnings --- tests/test_crawler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8c4bbe0d9..e811c5757 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -182,8 +182,12 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): 'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader' }) self.assertIsInstance(runner.spider_loader, CustomSpiderLoader) - self.assertEqual(len(w), 1) + is_one_warning = len(w) == 1 + if not is_one_warning: + for warning in w: + print(warning) self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message)) + self.assertTrue(is_one_warning) def test_crawl_rejects_spider_objects(self): with raises(ValueError): From dc8310e2929d228a224f12055a26cfef71d751cd Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 15:42:58 +0530 Subject: [PATCH 0376/1853] changed tests --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index feea54a54..81c1a4a9e 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -274,12 +274,13 @@ class FormRequestTest(RequestTest): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') - def test_formdata_overrides_querystring_duplicates(self): + def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) + self.assertNone(fs[b'c']) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 213b9eb879c4c6d8ae60ffa950688a38951b3349 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 15:59:38 +0530 Subject: [PATCH 0377/1853] Update test_http_request.py --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 81c1a4a9e..56c7d3d94 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertNone(fs[b'c']) + self.assertIsNone(fs[b'c']) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From ae856e8ba835d1d99510e9509a462b55a757e85f Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 16:21:52 +0530 Subject: [PATCH 0378/1853] corrected tests --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 56c7d3d94..55fa4ad22 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs[b'c']) + self.assertIsNone(fs.get([b'c'])) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 431f18a9a1a87ac8b789e3c260016e510ab48ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Feb 2019 13:22:38 +0100 Subject: [PATCH 0379/1853] Document FilesPipeline.file_path and ImagesPipeline.file_path --- docs/topics/media-pipeline.rst | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index c60b55391..c97b4c3c2 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -392,6 +392,36 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline + .. method:: file_path(request, response, info) + + This method is called once per downloaded item. It returns the + download path of the file originating from the specified + :class:`response `. + + In addition to ``response``, this method receives the original + :class:`request ` and + :class:`info `. + + You can override this method to customize the download path of each file. + + For example, if file URLs end like regular paths (e.g. + ``https://example.com/a/b/c/foo.png``), you can use the following + approach to download all files into the ``files`` folder with their + original filenames (e.g. ``files/foo.png``):: + + import os + from urllib.parse import urlparse + + from scrapy.pipelines.files import FilesPipeline + + class MyFilesPipeline(FilesPipeline): + + def file_path(self, request, response, info): + return 'files/' + os.path.basename(urlparse(request.url).path) + + By default the :meth:`file_path` method returns + ``full/.``. + .. method:: FilesPipeline.get_media_requests(item, info) As seen on the workflow, the pipeline will get the URLs of the images to @@ -475,6 +505,36 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. + .. method:: file_path(request, response, info) + + This method is called once per downloaded item. It returns the + download path of the file originating from the specified + :class:`response `. + + In addition to ``response``, this method receives the original + :class:`request ` and + :class:`info `. + + You can override this method to customize the download path of each file. + + For example, if file URLs end like regular paths (e.g. + ``https://example.com/a/b/c/foo.png``), you can use the following + approach to download all files into the ``files`` folder with their + original filenames (e.g. ``files/foo.png``):: + + import os + from urllib.parse import urlparse + + from scrapy.pipelines.images import ImagesPipeline + + class MyImagesPipeline(ImagesPipeline): + + def file_path(self, request, response, info): + return 'files/' + os.path.basename(urlparse(request.url).path) + + By default the :meth:`file_path` method returns + ``full/.``. + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, From 5f2ad5377e54ecdb3059891db4887eb09e56bfc6 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 16:46:15 +0530 Subject: [PATCH 0380/1853] fixed typo --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 55fa4ad22..89e4a8683 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs.get([b'c'])) + self.assertIsNone(fs.get(b'c')) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 2fd8b7c28c255148f6d7320cb97292ab2d569eff Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 27 Mar 2019 00:45:53 +0530 Subject: [PATCH 0381/1853] [MRG+1] redirect_reasons in Request.meta (#3687) --- docs/topics/downloader-middleware.rst | 16 ++++++++++++++-- docs/topics/request-response.rst | 1 + scrapy/downloadermiddlewares/redirect.py | 2 ++ tests/test_downloadermiddleware_redirect.py | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0d976077b..f2f3ef466 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -733,6 +733,17 @@ RedirectMiddleware The urls which the request goes through (while being redirected) can be found in the ``redirect_urls`` :attr:`Request.meta ` key. +.. reqmeta:: redirect_reasons + +The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the +``redirect_reasons`` :attr:`Request.meta ` key. For +example: ``[301, 302, 307, 'meta refresh']``. + +The format of a reason depends on the middleware that handled the corresponding +redirect. For example, :class:`RedirectMiddleware` indicates the triggering +response status code as an integer, while :class:`MetaRefreshMiddleware` +always uses the ``'meta refresh'`` string as reason. + The :class:`RedirectMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -796,8 +807,9 @@ settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` * :setting:`METAREFRESH_MAXDELAY` -This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect` -and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware` +This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, +:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described +for :class:`RedirectMiddleware` MetaRefreshMiddleware settings diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ca59b46d8..ac6fe6e3f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -299,6 +299,7 @@ Those are: * :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` +* :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`bindaddress` * :reqmeta:`dont_obey_robotstxt` diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 30cae3fee..cb59d3fd2 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -34,6 +34,8 @@ class BaseRedirectMiddleware(object): redirected.meta['redirect_ttl'] = ttl - 1 redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \ [request.url] + redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \ + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 74137b4cd..6c81c94ca 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -139,6 +139,16 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + def test_redirect_reasons(self): + req1 = Request('http://scrapytest.org/first') + rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301) + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301) + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.meta['redirect_reasons'], [301]) + self.assertEqual(req3.meta['redirect_reasons'], [301, 301]) + def test_spider_handling(self): smartspider = self.crawler._create_spider('smarty') smartspider.handle_httpstatus_list = [404, 301, 302] @@ -259,6 +269,15 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + def test_redirect_reasons(self): + req1 = Request('http://scrapytest.org/first') + rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected')) + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1')) + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh']) + self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh']) if __name__ == "__main__": unittest.main() From ce837b0f740e989520cce58d1606a528b598503e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Mar 2019 18:04:04 +0100 Subject: [PATCH 0382/1853] Update the documentation policies: Ask to use docstrings to document API members --- docs/contributing.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9b508e418..aac0f4496 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -165,18 +165,18 @@ Scrapy: Documentation policies ====================== -* **Don't** use docstrings for documenting classes, or methods which are - already documented in the official (sphinx) documentation. Alternatively, - **do** provide a docstring, but make sure sphinx documentation uses - autodoc_ extension to pull the docstring. For example, the - :meth:`ItemLoader.add_value` method should be either - documented only in the sphinx documentation (not as a docstring), or - it should have a docstring which is pulled to sphinx documentation using - autodoc_ extension. +For reference documentation of API members (classes, methods, etc.) use +docstrings and make sure that the Sphinx documentation uses the autodoc_ +extension to pull the docstrings. API reference documentation should be +IDE-friendly: short, to the point, and it may provide short examples. -* **Do** use docstrings for documenting functions not present in the official - (sphinx) documentation, such as functions from ``scrapy.utils`` package and - its sub-modules. +Other types of documentation, such as tutorials or topics, should be covered in +files within the ``docs/`` directory. This includes documentation that is +specific to an API member, but goes beyond API reference documentation. + +In any case, if something is covered in a docstring, use the autodoc_ +extension to pull the docstring into the documentation instead of duplicating +the docstring in files within the ``docs/`` directory. .. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html From 845bae6637239c859c9952c23f42902e36d10f6b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Mar 2019 08:49:19 +0000 Subject: [PATCH 0383/1853] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 37f7a8748..64c8883b1 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -42,7 +42,7 @@ efficient broad crawl. Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== -Default scrapy's scheduler priority queue is ``'queuelib.PriorityQueue'``. +Default scrapy's scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. It works best during single domain crawl. And it does not work well with crawling many different domains in parallel From 9c9bca4e1c7984089c44f3a44e7594e06307b12f Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Wed, 27 Mar 2019 18:29:48 +0530 Subject: [PATCH 0384/1853] make suggested changes. --- docs/topics/downloader-middleware.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f913b059d..dfbcdb8fa 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -349,7 +349,7 @@ HttpCacheMiddleware * :ref:`httpcache-storage-leveldb` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` - setting. Or you can also implement your own storage backend. + setting. Or you can also :ref:`implement your own storage backend. ` Scrapy ships with two HTTP cache policies: @@ -498,10 +498,10 @@ In order to use this storage backend: .. _httpcache-storage-custom: -Implementing custom cache storage backend -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Writing your own storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can implement custom cache storage backend by creating a Python class that +You can implement a cache storage backend by creating a Python class that defines the methods described below. .. module:: scrapy.extensions.httpcache @@ -510,14 +510,16 @@ defines the methods described below. .. method:: open_spider(spider) - This method gets called after a spider has been opened for crawling. + This method gets called after a spider has been opened for crawling. It handles + the :signal:`open_spider ` signal. :param spider: the spider which has been opened :type spider: :class:`~scrapy.spiders.Spider` object .. method:: close_spider(spider) - This method gets called after a spider has been closed. + This method gets called after a spider has been closed. It handles + the :signal:`close_spider ` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spiders.Spider` object @@ -539,7 +541,7 @@ defines the methods described below. :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object - :param request: corresponding request the spider generated + :param request: the corresponding request the spider generated :type request: :class:`~scrapy.http.Request` object :param response: the response to store in the cache @@ -547,7 +549,7 @@ defines the methods described below. In order to use your storage backend, set: -* :setting:`HTTPCACHE_STORAGE` to path of your custom storage class. +* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class. HTTPCache middleware settings From 8528f5065f99046b149b5e1901d6cbe5296f048a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 27 Mar 2019 14:42:26 -0300 Subject: [PATCH 0385/1853] [Doc] Update cb_kwargs example --- docs/topics/request-response.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b3f849540..61789be0f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -233,19 +233,19 @@ The following two examples show how to achieve this by using the :: - def parse_page1(self, response): - item = MyItem() - item['main_url'] = response.url - request = scrapy.Request("http://www.example.com/some_page.html", - callback=self.parse_page2) - request.cb_kwargs['item'] = item - request.cb_kwargs['foo'] = 'bar' + def parse(self, response): + request = scrapy.Request('http://www.example.com/index.html', + callback=self.parse_page2, + cb_kwargs=dict(main_url=response.url)) + request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback yield request - def parse_page2(self, response, item, foo): - item['other_url'] = response.url - item['foo'] = foo - yield item + def parse_page2(self, response, main_url, foo): + yield dict( + main_url=main_url, + other_url=response.url, + foo=foo, + ) .. _topics-request-response-ref-errbacks: From 70a4d93aa324fb276e60d641b37bdc6eb707b1cb Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 28 Mar 2019 10:40:41 -0300 Subject: [PATCH 0386/1853] Callback kwargs: more tests --- tests/spiders.py | 22 ++++++++++++++++++++++ tests/test_crawl.py | 21 ++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index a06985837..8c8d50ff5 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -39,6 +39,9 @@ class KeywordArgumentsSpider(MockServerSpider): yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) + yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data) + yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data) + yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data) def parse_first(self, response, key, number): self.checks.append(key == 'value') @@ -66,6 +69,25 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(response.url.endswith('/no_kwargs')) self.crawler.stats.inc_value('boolean_checks') + def parse_default(self, response, key, number=None, default=99): + self.checks.append(response.url.endswith('/default')) + self.checks.append(key == 'value') + self.checks.append(number == 123) + self.checks.append(default == 99) + self.crawler.stats.inc_value('boolean_checks', 4) + + def parse_takes_less(self, response, key): + """ + Should raise + TypeError: parse_takes_less() got an unexpected keyword argument 'number' + """ + + def parse_takes_more(self, response, key, number, other): + """ + Should raise + TypeError: parse_takes_more() missing 1 required positional argument: 'other' + """ + class FollowAllSpider(MetaSpider): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9a39b8cb4..2b3e56ee9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -4,6 +4,7 @@ import logging from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase +import six from scrapy.http import Request from scrapy.crawler import CrawlerRunner @@ -15,6 +16,8 @@ from tests.mockserver import MockServer class CrawlTestCase(TestCase): + maxDiff = None + def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -26,9 +29,25 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_callback_kwargs(self): crawler = self.runner.create_crawler(KeywordArgumentsSpider) - yield crawler.crawl(mockserver=self.mockserver) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) self.assertTrue(all(crawler.spider.checks)) self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks')) + # check exceptions for argument mismatch + exceptions = {} + for line in log.records: + for key in ('takes_less', 'takes_more'): + if key in line.getMessage(): + exceptions[key] = line + self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) + self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") + self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) + # py2 and py3 messages are different + exc_message = str(exceptions['takes_more'].exc_info[1]) + if six.PY2: + self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)") + elif six.PY3: + self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'") @defer.inlineCallbacks def test_follow_all(self): From 3efe3bea1cbb5ae83c024fc6dc8e1776a47a345f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 28 Mar 2019 14:16:03 -0300 Subject: [PATCH 0387/1853] Update docs about cb_kwargs and meta --- docs/topics/request-response.rst | 41 ++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index dd0db8156..05ca8d6c1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -215,24 +215,12 @@ Example:: In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. -The following two examples show how to achieve this by using the -:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively:: - - def parse_page1(self, response): - item = MyItem() - item['main_url'] = response.url - request = scrapy.Request("http://www.example.com/some_page.html", - callback=self.parse_page2) - request.meta['item'] = item - yield request - - def parse_page2(self, response): - item = response.meta['item'] - item['other_url'] = response.url - yield item +The following example shows how to achieve this by using the +:attr:`Request.cb_kwargs` attribute: :: + # pass information to the next callback using the Request.cb_kwargs attribute def parse(self, response): request = scrapy.Request('http://www.example.com/index.html', callback=self.parse_page2, @@ -247,6 +235,29 @@ The following two examples show how to achieve this by using the foo=foo, ) +.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. + Prior to that, :attr:`Request.meta` was the recommended option for passing + information around callbacks. However, after ``1.7`` :attr:`Request.cb_kwargs` + became the preferred way of passing user information, leaving :attr:`Request.meta` + to be used by internal components like spider or downloader middlewares. + The following example, which uses :attr:`Request.meta`, is only kept for historical + reasons. + +:: + + # pass information to the next callback using the Request.meta attribute + def parse_page1(self, response): + item = MyItem() + item['main_url'] = response.url + request = scrapy.Request("http://www.example.com/some_page.html", + callback=self.parse_page2) + request.meta['item'] = item + yield request + + def parse_page2(self, response): + item = response.meta['item'] + item['other_url'] = response.url + yield item .. _topics-request-response-ref-errbacks: From e8af6331b5ff62d71ff80eddcc52b85c25482c0e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 28 Mar 2019 14:56:31 -0300 Subject: [PATCH 0388/1853] Add cb_kwargs option to the parse command --- docs/topics/commands.rst | 3 +++ scrapy/commands/parse.py | 37 ++++++++++++++++++++++++------------- tests/test_command_parse.py | 14 ++++++++++++++ 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 97f8311de..6644d65e4 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -461,6 +461,9 @@ Supported options: * ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' +* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback. + This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 69418a478..2486f3f23 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,12 +51,13 @@ class Command(ScrapyCommand): help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_option("--cb_kwargs", dest="cb_kwargs", + help="inject extra cb_kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") - @property def max_level(self): levels = list(self.items.keys()) + list(self.requests.keys()) @@ -111,10 +112,11 @@ class Command(ScrapyCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, cb): + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} items, requests = [], [] - for x in iterate_spider_output(cb(response)): + for x in iterate_spider_output(callback(response, **cb_kwargs)): if isinstance(x, (BaseItem, dict)): items.append(x) elif isinstance(x, Request): @@ -142,8 +144,7 @@ class Command(ScrapyCommand): else: self.spidercls = spidercls_for_request(spider_loader, Request(url)) if not self.spidercls: - logger.error('Unable to find spider for: %(url)s', - {'url': url}) + logger.error('Unable to find spider for: %(url)s', {'url': url}) # Request requires callback argument as callable or None, not string request = Request(url, None) @@ -160,7 +161,7 @@ class Command(ScrapyCommand): {'url': url}) def prepare_request(self, spider, request, opts): - def callback(response): + def callback(response, **cb_kwargs): # memorize first request if not self.first_response: self.first_response = response @@ -175,7 +176,7 @@ class Command(ScrapyCommand): if not cb: logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', - {'url': response.url, 'spider': spider.name}) + {'url': response.url, 'spider': spider.name}) return else: cb = 'parse' @@ -192,7 +193,7 @@ class Command(ScrapyCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb) + items, requests = self.run_callback(response, cb, cb_kwargs) if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc for item in items: @@ -207,10 +208,14 @@ class Command(ScrapyCommand): req.callback = callback return requests - #update request meta if any extra meta was passed through the --meta/-m opts. + # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: request.meta.update(opts.meta) + # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option. + if opts.cb_kwargs: + request.cb_kwargs.update(opts.cb_kwargs) + request.meta['_depth'] = 1 request.meta['_callback'] = request.callback request.callback = callback @@ -221,23 +226,29 @@ class Command(ScrapyCommand): self.process_spider_arguments(opts) self.process_request_meta(opts) + self.process_request_cb_kwargs(opts) def process_spider_arguments(self, opts): - try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) def process_request_meta(self, opts): - if opts.meta: try: opts.meta = json.loads(opts.meta) except ValueError: - raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \ - "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " + "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + def process_request_cb_kwargs(self, opts): + if opts.cb_kwargs: + try: + opts.cb_kwargs = json.loads(opts.cb_kwargs) + except ValueError: + raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. " + "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False) def run(self, args, opts): # parse arguments diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 02037b866..1404005fb 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -43,6 +43,12 @@ class MySpider(scrapy.Spider): else: self.logger.debug('It Works!') + def parse_request_with_cb_kwargs(self, response, foo=None, key=None): + if foo == 'bar' and key == 'value': + self.logger.debug('It Works!') + else: + self.logger.debug('It Does Not Work :(') + def parse_request_without_meta(self, response): foo = response.meta.get('foo', 'bar') @@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_request_with_cb_kwargs(self): + raw_json_string = '{"foo" : "bar", "key": "value"}' + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--cb_kwargs', raw_json_string, + '-c', 'parse_request_with_cb_kwargs', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_without_meta(self): From 8fb077694fcaa50a8625c8e2e8d0068add2b056d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 28 Mar 2019 15:18:00 -0300 Subject: [PATCH 0389/1853] Request.cb_kwargs: Update docs --- docs/topics/debug.rst | 13 +++++-------- docs/topics/jobs.rst | 9 +++++---- docs/topics/leaks.rst | 14 ++++++++------ docs/topics/request-response.rst | 13 ++++++------- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index f93aa2c72..0aaad0c77 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -28,16 +28,15 @@ Consider the following scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item}) + yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) - def parse_details(self, response): - item = response.meta['item'] + def parse_details(self, response, item): # populate more `item` fields return item Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a partially populated item. @@ -100,8 +99,7 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see from scrapy.shell import inspect_response - def parse_details(self, response): - item = response.meta.get('item', None) + def parse_details(self, response, item=None): if item: # populate more `item` fields return item @@ -134,8 +132,7 @@ Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be available in all future runs should they be necessary again:: - def parse_details(self, response): - item = response.meta.get('item', None) + def parse_details(self, response, item=None): if item: # populate more `item` fields return item diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 1a5d52487..9fd311c69 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -81,7 +81,8 @@ So, for example, this won't work:: def some_callback(self, response): somearg = 'test' - return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg)) + return scrapy.Request('http://www.example.com', + callback=lambda r: self.other_callback(r, somearg)) def other_callback(self, response, somearg): print("the argument passed is: %s" % somearg) @@ -90,10 +91,10 @@ But this will:: def some_callback(self, response): somearg = 'test' - return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg}) + return scrapy.Request('http://www.example.com', + callback=self.other_callback, cb_kwargs={'somearg': somearg}) - def other_callback(self, response): - somearg = response.meta['somearg'] + def other_callback(self, response, somearg): print("the argument passed is: %s" % somearg) If you wish to log the requests that couldn't be serialized, you can set the diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index af14d14e8..8278e9849 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -27,10 +27,11 @@ Common causes of memory leaks It happens quite often (sometimes by accident, sometimes on purpose) that the Scrapy developer passes objects referenced in Requests (for example, using the -:attr:`~scrapy.http.Request.meta` attribute or the request callback function) -and that effectively bounds the lifetime of those referenced objects to the -lifetime of the Request. This is, by far, the most common cause of memory leaks -in Scrapy projects, and a quite difficult one to debug for newcomers. +:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta` +attributes or the request callback function) and that effectively bounds the +lifetime of those referenced objects to the lifetime of the Request. This is, +by far, the most common cause of memory leaks in Scrapy projects, and a quite +difficult one to debug for newcomers. In big projects, the spiders are typically written by different people and some of those spiders could be "leaking" and thus affecting the rest of the other @@ -48,7 +49,8 @@ Too Many Requests? By default Scrapy keeps the request queue in memory; it includes :class:`~scrapy.http.Request` objects and all objects -referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`). +referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs` +and :attr:`~scrapy.http.Request.meta`). While not necessarily a leak, this can take a lot of memory. Enabling :ref:`persistent job queue ` could help keeping memory usage in control. @@ -101,7 +103,7 @@ Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, - callback=self.parse, meta={referer: response}) + callback=self.parse, cb_kwargs={'referer': response}) That line is passing a response reference inside a request which effectively ties the response lifetime to the requests' one, and that would definitely diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 05ca8d6c1..f299c2cff 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -186,12 +186,12 @@ Request objects Return a new Request which is a copy of this Request. See also: :ref:`topics-request-response-ref-request-callback-arguments`. - .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback]) + .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs]) Return a Request object with the same members, except for those members given new values by whichever keyword arguments are specified. The - attribute :attr:`Request.meta` is copied by default (unless a new value - is given in the ``meta`` argument). See also + :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are copied by default + (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. .. _topics-request-response-ref-request-callback-arguments: @@ -237,11 +237,10 @@ The following example shows how to achieve this by using the .. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. Prior to that, :attr:`Request.meta` was the recommended option for passing - information around callbacks. However, after ``1.7`` :attr:`Request.cb_kwargs` + information around callbacks. However, after ``1.7``, using :attr:`Request.cb_kwargs` became the preferred way of passing user information, leaving :attr:`Request.meta` - to be used by internal components like spider or downloader middlewares. - The following example, which uses :attr:`Request.meta`, is only kept for historical - reasons. + to be populated by internal components like spider or downloader middlewares. + The following :attr:`Request.meta` example is only kept for historical reasons. :: From 46b9ab0c58354deb1045c20f3bc061526d69f356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:28:36 +0000 Subject: [PATCH 0390/1853] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 64c8883b1..68a24a4d2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -42,7 +42,7 @@ efficient broad crawl. Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== -Default scrapy's scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. +Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. It works best during single domain crawl. And it does not work well with crawling many different domains in parallel From e3df6be360a58f016e31d5bfa2e04cd2e5d1965b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:28:52 +0000 Subject: [PATCH 0391/1853] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 68a24a4d2..b149d7f4a 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -43,7 +43,7 @@ Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. -It works best during single domain crawl. And it does not work well with crawling +It works best during single-domain crawl. It does not work well with crawling many different domains in parallel To apply recommended priority queue use:: From bd228f1d962c7f4759536d8cda278857de7d5234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:29:04 +0000 Subject: [PATCH 0392/1853] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index b149d7f4a..a01f28248 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -46,7 +46,7 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ It works best during single-domain crawl. It does not work well with crawling many different domains in parallel -To apply recommended priority queue use:: +To apply the recommended priority queue use:: SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' From 1ee99e1f4240af6a7a72fe7c58b89d7bce1cd09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:29:15 +0000 Subject: [PATCH 0393/1853] Update docs/topics/settings.rst Co-Authored-By: whalebot-helmsman --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ed94146f4..4a5439bfc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1157,7 +1157,7 @@ SCHEDULER_PRIORITY_QUEUE ------------------------ Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` -Type of priority queue used by scheduler. Another available type is +Type of priority queue used by the scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. ``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different From 2b4bcfaf494073520e84bbf301d5141a2e19a3e6 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:30:26 +0000 Subject: [PATCH 0394/1853] remove comment --- scrapy/core/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index c385fafe1..9d0258db2 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -57,7 +57,6 @@ class Scheduler(object): dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) if pqclass is PriorityQueue: - # backwards compatibility warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" " is no longer supported because of API changes; " "please use 'scrapy.pqueues.ScrapyPriorityQueue'", From 554d8728227a9ea96e5ea3a8a4fd782d42fdbd66 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:31:15 +0000 Subject: [PATCH 0395/1853] remove spacing --- scrapy/core/scheduler.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 9d0258db2..d87d2ffdc 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -77,13 +77,8 @@ class Scheduler(object): def open(self, spider): self.spider = spider - - # in-memory PriorityQueue instance self.mqs = self._mq() - - # on-disk PriorityQueue instance self.dqs = self._dq() if self.dqdir else None - return self.df.open() def close(self, reason): From f08f841d0bebd097358889c0c98f83f051828f15 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:35:49 +0000 Subject: [PATCH 0396/1853] remove small single use method --- scrapy/core/scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d87d2ffdc..975aede0c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -101,7 +101,7 @@ class Scheduler(object): return True def next_request(self): - request = self._mqpop() + request = self.mqs.pop() if request: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: @@ -141,9 +141,6 @@ class Scheduler(object): if self.dqs: return self.dqs.pop() - def _mqpop(self): - return self.mqs.pop() - def _newmq(self, priority): """ Factory for creating memory queues. """ return self.mqclass() From ef743983a98ae0891abf9aca4c9b19cb44861c49 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:38:13 +0000 Subject: [PATCH 0397/1853] change wording --- docs/topics/broad-crawls.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index a01f28248..6e50c0bc7 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -39,7 +39,7 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. -Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` +Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. @@ -96,7 +96,7 @@ When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you should not use ``DEBUG`` log level when preforming large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may be +production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. To set the log level use:: From 1c6733454e14a3c237ed602b65ae5e0a8a78dee5 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:44:55 +0000 Subject: [PATCH 0398/1853] added underlines --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 6e50c0bc7..b887b98af 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -40,7 +40,7 @@ concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` -============================================== +================================================= Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. It works best during single-domain crawl. It does not work well with crawling From f5e0b6b89ace437af850e0225651329101a59862 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Mar 2019 14:03:26 -0300 Subject: [PATCH 0399/1853] parse command: rename cb_kwargs option to cbkwargs --- scrapy/commands/parse.py | 18 +++++++++--------- tests/test_command_parse.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2486f3f23..e948d6406 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,8 +51,8 @@ class Command(ScrapyCommand): help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_option("--cb_kwargs", dest="cb_kwargs", - help="inject extra cb_kwargs into the Request, it must be a valid raw json string") + parser.add_option("--cbkwargs", dest="cbkwargs", + help="inject extra cbkwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", @@ -212,9 +212,9 @@ class Command(ScrapyCommand): if opts.meta: request.meta.update(opts.meta) - # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option. - if opts.cb_kwargs: - request.cb_kwargs.update(opts.cb_kwargs) + # update cb_kwargs if any extra values were was passed through the --cbkwargs option. + if opts.cbkwargs: + request.cb_kwargs.update(opts.cbkwargs) request.meta['_depth'] = 1 request.meta['_callback'] = request.callback @@ -243,12 +243,12 @@ class Command(ScrapyCommand): "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) def process_request_cb_kwargs(self, opts): - if opts.cb_kwargs: + if opts.cbkwargs: try: - opts.cb_kwargs = json.loads(opts.cb_kwargs) + opts.cbkwargs = json.loads(opts.cbkwargs) except ValueError: - raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. " - "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False) + raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " + "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False) def run(self, args, opts): # parse arguments diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 1404005fb..c18a6ce9f 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -130,7 +130,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--cb_kwargs', raw_json_string, + '--cbkwargs', raw_json_string, '-c', 'parse_request_with_cb_kwargs', self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) From ccb56a317ee249978496e918ee5b74d83c2d7199 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Mar 2019 14:12:26 -0300 Subject: [PATCH 0400/1853] Update docs about cb_kwargs and meta --- docs/topics/request-response.rst | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f299c2cff..4e81ce878 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -190,8 +190,8 @@ Request objects Return a Request object with the same members, except for those members given new values by whichever keyword arguments are specified. The - :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are copied by default - (unless new values are given as arguments). See also + :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow + copied by default (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. .. _topics-request-response-ref-request-callback-arguments: @@ -220,7 +220,6 @@ The following example shows how to achieve this by using the :: - # pass information to the next callback using the Request.cb_kwargs attribute def parse(self, response): request = scrapy.Request('http://www.example.com/index.html', callback=self.parse_page2, @@ -236,27 +235,10 @@ The following example shows how to achieve this by using the ) .. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, :attr:`Request.meta` was the recommended option for passing - information around callbacks. However, after ``1.7``, using :attr:`Request.cb_kwargs` - became the preferred way of passing user information, leaving :attr:`Request.meta` - to be populated by internal components like spider or downloader middlewares. - The following :attr:`Request.meta` example is only kept for historical reasons. - -:: - - # pass information to the next callback using the Request.meta attribute - def parse_page1(self, response): - item = MyItem() - item['main_url'] = response.url - request = scrapy.Request("http://www.example.com/some_page.html", - callback=self.parse_page2) - request.meta['item'] = item - yield request - - def parse_page2(self, response): - item = response.meta['item'] - item['other_url'] = response.url - yield item + Prior to that, using :attr:`Request.meta` was recommended for passing + information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs` + became the preferred way for handling user information, leaving :attr:`Request.meta` + for communication with components like middlewares and extensions. .. _topics-request-response-ref-errbacks: From 294ef51bb24782a0527892ea93bb4876daa7ca50 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Mar 2019 16:12:55 -0300 Subject: [PATCH 0401/1853] parse command: update docs about passing callback keyword arguments --- docs/topics/commands.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6644d65e4..a93bee06b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -461,8 +461,8 @@ Supported options: * ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' -* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback. - This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}' +* ``--cbkwargs``: additional keyword arguments that will be passed to the callback. + This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}' * ``--pipelines``: process items through pipelines From 0522fe35c334141e90741644fec368cdbd12044e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Mar 2019 16:15:34 -0300 Subject: [PATCH 0402/1853] parse command: improve option description --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index e948d6406..d4f2234b0 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -52,7 +52,7 @@ class Command(ScrapyCommand): parser.add_option("-m", "--meta", dest="meta", help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra cbkwargs into the Request, it must be a valid raw json string") + help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", From 07ff9248a5fd2eac4f53da92766dcb5a7ca48569 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 1 Apr 2019 12:31:26 -0300 Subject: [PATCH 0403/1853] [Docs] CrawlSpider: add note about link text --- docs/topics/spiders.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8c4049f85..5417ef129 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -377,7 +377,10 @@ Crawling rules .. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None) ``link_extractor`` is a :ref:`Link Extractor ` object which - defines how links will be extracted from each crawled page. + defines how links will be extracted from each crawled page. Each produced link will + be used to generate a :class:`~scrapy.http.Request` object, which will contain the + link's text in its ``meta`` attribute. + The link text can be accessed from the callback method though ``response.meta['link_text']`` ``callback`` is a callable or a string (in which case a method from the spider object with that name will be used) to be called for each link extracted with @@ -438,6 +441,7 @@ Let's now take a look at an example CrawlSpider with rules:: item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') item['name'] = response.xpath('//td[@id="item_name"]/text()').get() item['description'] = response.xpath('//td[@id="item_description"]/text()').get() + item['link_text'] = response.meta['link_text'] return item From 8ebbc731b2bdf8e2a2b5a2f0673da838369f31b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 1 Apr 2019 16:15:03 -0300 Subject: [PATCH 0404/1853] [Docs] Rephrase Rule docs --- docs/topics/spiders.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5417ef129..7290bb844 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -380,13 +380,13 @@ Crawling rules defines how links will be extracted from each crawled page. Each produced link will be used to generate a :class:`~scrapy.http.Request` object, which will contain the link's text in its ``meta`` attribute. - The link text can be accessed from the callback method though ``response.meta['link_text']`` ``callback`` is a callable or a string (in which case a method from the spider object with that name will be used) to be called for each link extracted with - the specified link_extractor. This callback receives a response as its first - argument and must return a list containing :class:`~scrapy.item.Item` and/or - :class:`~scrapy.http.Request` objects (or any subclass of them). + the specified link extractor. This callback receives a :class:`~scrapy.http.Response` + as its first argument and must return either a single instance or an iterable of + :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects + (or any subclass of them). .. warning:: When writing crawl spider rules, avoid using ``parse`` as callback, since the :class:`CrawlSpider` uses the ``parse`` method From 7a38623cecc6c60d7ffe14c75d1fe679bb04b774 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 1 Apr 2019 17:09:49 -0300 Subject: [PATCH 0405/1853] [Docs] Clarify comment about meta dictionary --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 7290bb844..3cd051cdf 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -379,7 +379,7 @@ Crawling rules ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. Each produced link will be used to generate a :class:`~scrapy.http.Request` object, which will contain the - link's text in its ``meta`` attribute. + link's text in its ``meta`` dictionary (under the ``link_text`` key). ``callback`` is a callable or a string (in which case a method from the spider object with that name will be used) to be called for each link extracted with From 7acf4eec792f155a8b0e92c3bb1efa1ff5882ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Apr 2019 18:36:03 +0200 Subject: [PATCH 0406/1853] Deprecate the scrapy.utils.gz.is_gzipped function --- scrapy/utils/gz.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index ec3949651..b3fb16b1e 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -9,6 +9,9 @@ from gzip import GzipFile import six import re +from scrapy.utils.decorators import deprecated + + # - Python>=3.5 GzipFile's read() has issues returning leftover # uncompressed data when input is corrupted # (regression or bug-fix compared to Python 3.4) @@ -53,6 +56,7 @@ def gunzip(data): _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search +@deprecated def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') From 6336e1d1f31da8611a0b63dcb536529e9027d51b Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 00:54:46 -0400 Subject: [PATCH 0407/1853] grammar fix - delete unneeded apostrophe in "lets" --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 41e61542a..b2f952fe2 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -511,7 +511,7 @@ We can try extracting it in the shell:: 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, -Scrapy supports a CSS extension that let's you select the attribute contents, +Scrapy supports a CSS extension that lets you select the attribute contents, like this:: >>> response.css('li.next a::attr(href)').get() From d711ecfc18a01084f74bb2b9dc01c8bcb4772580 Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 00:56:51 -0400 Subject: [PATCH 0408/1853] grammar fix - delete extra word "shell" --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index b2f952fe2..fc10adbe1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -205,7 +205,7 @@ Extracting data --------------- The best way to learn how to extract data with Scrapy is trying selectors -using the shell :ref:`Scrapy shell `. Run:: +using the :ref:`Scrapy shell `. Run:: scrapy shell 'http://quotes.toscrape.com/page/1/' From 77e3695686d9a46841778248f4c1a2da336b54f7 Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 01:04:59 -0400 Subject: [PATCH 0409/1853] grammar fix - add apostrophe-s to browser --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fc10adbe1..a97f96801 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -296,7 +296,7 @@ expressions`_:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools to inspect the HTML and come up +You can use your browser's developer tools to inspect the HTML and come up with a selector (see section about :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for From a101d5fe5c215bfbb09732fb506d4cd016624d80 Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 01:12:20 -0400 Subject: [PATCH 0410/1853] text edit - delete 2 extra words --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a97f96801..8bd2d27dd 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -297,7 +297,7 @@ expressions`_:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. You can use your browser's developer tools to inspect the HTML and come up -with a selector (see section about :ref:`topics-developer-tools`). +with a selector (see :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. From 3a493b60661760b26ebc9dd2f4c5c7e4b8df93c9 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Fri, 5 Apr 2019 11:52:00 +0200 Subject: [PATCH 0411/1853] fix: do not catch system exceptions like KeyboardInterrupt --- scrapy/contracts/__init__.py | 2 +- scrapy/core/spidermw.py | 2 +- scrapy/utils/defer.py | 4 ++-- scrapy/utils/misc.py | 2 +- tests/mockserver.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 259220a72..536bbdafb 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -94,7 +94,7 @@ class ContractsManager(object): try: output = cb(response) output = list(iterate_spider_output(output)) - except: + except Exception: case = _create_testcase(method, 'callback') results.addError(case, sys.exc_info()) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index e07f76bdf..b5f9837ff 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -49,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager): .format(fname(method), type(result))) except _InvalidOutput: raise - except: + except Exception: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bcf209511..69d621830 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -48,7 +48,7 @@ def mustbe_deferred(f, *args, **kw): # exception in Scrapy - see #125 except IgnoreRequest as e: return defer_fail(failure.Failure(e)) - except: + except Exception: return defer_fail(failure.Failure()) else: return defer_result(result) @@ -102,5 +102,5 @@ def iter_errback(iterable, errback, *a, **kw): yield next(it) except StopIteration: break - except: + except Exception: errback(failure.Failure(), *a, **kw) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 6de36d45c..ddaa7f7bf 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -86,7 +86,7 @@ def extract_regex(regex, text, encoding='utf-8'): try: strings = [regex.search(text).group('extract')] # named group - except: + except Exception: strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) diff --git a/tests/mockserver.py b/tests/mockserver.py index bf62fe907..3fa4bc0f0 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -177,7 +177,7 @@ class Root(Resource): try: from tests import tests_datadir self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) - except: + except Exception: pass self.putChild(b"redirect-to", RedirectTo()) From 35ce92a4199b29a30e58026880b45002a1e4591e Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 5 Apr 2019 11:43:21 -0300 Subject: [PATCH 0412/1853] fix typo (Response -> Request) check docs for more information https://github.com/scrapy/scrapy/blob/b5c552d17ff9e9629434712c3d0595c02853bcfc/docs/topics/spider-middleware.rst --- scrapy/templates/project/module/middlewares.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 5debe1cd2..97b5db2e1 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -39,7 +39,7 @@ class ${ProjectName}SpiderMiddleware(object): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. - # Should return either None or an iterable of Response, dict + # Should return either None or an iterable of Request, dict # or Item objects. pass From a8f83ab9675ec4f0bcb90b2eb3f06c593f32732f Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 6 Apr 2019 14:58:32 +0200 Subject: [PATCH 0413/1853] doc: document LOGSTATS_INTERVAL setting --- docs/topics/settings.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 062c4b2ca..145dcc136 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -897,6 +897,16 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. setting:: LOGSTATS_INTERVAL + +LOGSTATS_INTERVAL +----------------- + +Default: ``60.0`` + +The interval (in seconds) between each logging printout of the stats +by :class:`~extensions.logstats.LogStats`. + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED From aa46e1995cd5cb1099aba17535372b538bd656b3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Sun, 7 Apr 2019 00:33:40 +0530 Subject: [PATCH 0414/1853] [MRG+1] Show elapsed time in statscollector (#3638) * Update corestats.py * Update corestats.py * corrected tests * Update corestats.py * Update scrapy/extensions/corestats.py --- scrapy/extensions/corestats.py | 6 +++++- tests/test_closespider.py | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 3d9a307b7..8cc5e18ac 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -24,7 +24,11 @@ class CoreStats(object): self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) def spider_closed(self, spider, reason): - self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) + finish_time = datetime.datetime.utcnow() + elapsed_time = finish_time - self.stats.get_value('start_time') + elapsed_time_seconds = elapsed_time.total_seconds() + self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider) + self.stats.set_value('finish_time', finish_time, spider=spider) self.stats.set_value('finish_reason', reason, spider=spider) def item_scraped(self, item, spider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 0eb1b7944..4a56425b7 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -53,9 +53,5 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') - stats = crawler.stats - start = stats.get_value('start_time') - stop = stats.get_value('finish_time') - diff = stop - start - total_seconds = diff.seconds + diff.microseconds + total_seconds = crawler.stats.get_value('elapsed_time_seconds') self.assertTrue(total_seconds >= close_on) From e6048d55f905ca0be2b32f9b566d257f34752c71 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 9 Apr 2019 17:34:20 +0530 Subject: [PATCH 0415/1853] changes parameter name --- scrapy/extensions/httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 1b5e05b1b..35c77add8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -31,7 +31,7 @@ class DummyPolicy(object): def should_cache_response(self, response, request): return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, response, request): + def is_cached_response_fresh(self, cachedresponse, request): return True def is_cached_response_valid(self, cachedresponse, response, request): From 4cfdc14974313f23d7bb8be9311195f9cfa74968 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 9 Apr 2019 17:52:02 +0530 Subject: [PATCH 0416/1853] fixes a link in comment --- scrapy/extensions/httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 1b5e05b1b..03bba1530 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -70,7 +70,7 @@ class RFC2616Policy(object): return True def should_cache_response(self, response, request): - # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) From d27c2c68ba4f201c18255a6ec8a735f140b13773 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 12:56:50 +0200 Subject: [PATCH 0417/1853] Wrap scrapy check in environment --- scrapy/commands/check.py | 21 +++++++++++---------- scrapy/utils/misc.py | 27 +++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b8a9ef989..b29587fa7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, set_environ from scrapy.utils.conf import build_component_list @@ -68,16 +68,17 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - for spidername in args or spider_loader.list(): - spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + with set_environ(SCRAPY_CHECK=True): + for spidername in args or spider_loader.list(): + spidercls = spider_loader.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) - tested_methods = conman.tested_methods_from_spidercls(spidercls) - if opts.list: - for method in tested_methods: - contract_reqs[spidercls.name].append(method) - elif tested_methods: - self.crawler_process.crawl(spidercls) + tested_methods = conman.tested_methods_from_spidercls(spidercls) + if opts.list: + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ddaa7f7bf..7a2cd18ea 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,6 +1,8 @@ """Helper functions which don't fit anywhere else""" +import os import re import hashlib +from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -10,7 +12,6 @@ from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem - _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes @@ -40,7 +41,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot+1:] + module, name = path[:dot], path[dot + 1:] mod = import_module(module) try: @@ -85,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): @@ -142,3 +143,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return objcls.from_settings(settings, *args, **kwargs) else: return objcls(*args, **kwargs) + + +@contextmanager +def set_environ(**kwargs): + """Temporarily set environment variables inside the context manager and + fully restore previous environment afterwards + """ + + original_env = {k: os.environ.get(k) for k in kwargs} + os.environ.update(kwargs) + try: + yield + finally: + for k, v in original_env: + if v is None: + del os.environ[k] + else: + os.environ[k] = v From 50730ed2280dec6384986d34999cd277d7568ff9 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 13:01:01 +0200 Subject: [PATCH 0418/1853] Try it with a string --- scrapy/commands/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b29587fa7..ab73e85e7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -68,7 +68,7 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - with set_environ(SCRAPY_CHECK=True): + with set_environ(SCRAPY_CHECK='true'): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) spidercls.start_requests = lambda s: conman.from_spider(s, result) From 07adca34e1378b11dff9e3f11d3760c54f5fa1ef Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 13:01:46 +0200 Subject: [PATCH 0419/1853] Fix 'Too many values to unpack' --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7a2cd18ea..cdd5a11c9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -156,7 +156,7 @@ def set_environ(**kwargs): try: yield finally: - for k, v in original_env: + for k, v in original_env.items(): if v is None: del os.environ[k] else: From fbb42fe14ed23aaba37b42d12c7adbf513f9089e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 17 Apr 2019 08:25:22 +0200 Subject: [PATCH 0420/1853] Cover PEP 257 in the documentation policies --- docs/contributing.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index aac0f4496..c31a17609 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -167,8 +167,9 @@ Documentation policies For reference documentation of API members (classes, methods, etc.) use docstrings and make sure that the Sphinx documentation uses the autodoc_ -extension to pull the docstrings. API reference documentation should be -IDE-friendly: short, to the point, and it may provide short examples. +extension to pull the docstrings. API reference documentation should follow +docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point, +and it may provide short examples. Other types of documentation, such as tutorials or topics, should be covered in files within the ``docs/`` directory. This includes documentation that is @@ -237,5 +238,6 @@ And their unit-tests are in:: .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues +.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/send-pull-requests/ .. _tox: https://pypi.python.org/pypi/tox From 5a6fb3daa6e6a15effe9377dbcc85e67bec9aec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 17:10:16 +0100 Subject: [PATCH 0421/1853] Use pytest-xdist --- docs/contributing.rst | 26 ++++++++++++++++++++++++++ tests/requirements-py2.txt | 5 +++-- tests/requirements-py3.txt | 5 +++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index aac0f4496..b462ae331 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -205,6 +205,29 @@ To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` +To run the tests on a specific tox_ environment, use ``-e `` with an +environment name from ``tox.ini``. For example, to run the tests with Python +3.6 use:: + + tox -e py36 + +You can also specify a comma-separated list of environmets, and use `tox’s +parallel mode`_ to run the tests on multiple environments in parallel:: + + tox -e py27,py36 -p auto + +To pass command-line options to pytest_, add them after ``--`` in your call to +tox_. Using ``--`` overrides the default positional arguments defined in +``tox.ini``, so you must include those default positional arguments +(``scrapy tests``) after ``--`` as well:: + + tox -- scrapy tests -x # stop after first failure + +You can also use the `pytest-xdist`_ plugin. For example, to run all tests on +the Python 3.6 tox_ environment using all your CPU cores:: + + tox -e py36 -- scrapy tests -n auto + To see coverage report install `coverage`_ (``pip install coverage``) and run: ``coverage report`` @@ -238,4 +261,7 @@ And their unit-tests are in:: .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _pull request: https://help.github.com/send-pull-requests/ +.. _pytest: https://docs.pytest.org/en/latest/usage.html +.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html .. _tox: https://pypi.python.org/pypi/tox +.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index 790f29d34..be809b151 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -2,9 +2,10 @@ mock mitmproxy==0.10.1 netlib==0.10.1 -pytest==2.9.2 +pytest +pytest-cov pytest-twisted -pytest-cov==2.2.1 +pytest-xdist jmespath brotlipy testfixtures diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 7c1aacd81..ed7bf0be0 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,7 @@ -pytest==3.6.3 +pytest +pytest-cov pytest-twisted -pytest-cov==2.5.1 +pytest-xdist testfixtures jmespath leveldb; sys_platform != "win32" From 29739989478bb1b10467b036df91326071b1ffbc Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 14:50:02 +0200 Subject: [PATCH 0422/1853] Add set_environ test --- tests/test_utils_misc/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index fcb7772ab..738120a0b 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,7 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ from tests import mock @@ -130,5 +130,12 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(ValueError): create_instance(m, None, None) + def test_set_environ(self): + assert os.environ.get('some_test_environ') is None + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') is None + + if __name__ == "__main__": unittest.main() From 6d52708579be05c29c58d6ccc63486f761466d18 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 15:19:23 +0200 Subject: [PATCH 0423/1853] Add reset case --- tests/test_utils_misc/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 738120a0b..eeb995e48 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -136,6 +136,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') is None + os.environ['some_test_environ'] = 'test' + assert os.environ.get('some_test_environ') == 'test' + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') == 'test' + if __name__ == "__main__": unittest.main() From 935387aaea2f0bdc28504b40dfc9ccbbb437bec4 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 22:10:23 +0200 Subject: [PATCH 0424/1853] Revert some non-changes --- scrapy/utils/misc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index cdd5a11c9..f51012e3d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -41,7 +41,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot + 1:] + module, name = path[:dot], path[dot+1:] mod = import_module(module) try: @@ -86,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): From 7809c0b14e3ad62aea8e62c7309997ecb64fbbf1 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Sat, 20 Apr 2019 09:25:01 +0200 Subject: [PATCH 0425/1853] Revert another non-change comment --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f51012e3d..b2164d4a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -88,7 +88,7 @@ def extract_regex(regex, text, encoding='utf-8'): try: strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): From 122ca6211935039825aff900e98bc1fbdb4dc0d6 Mon Sep 17 00:00:00 2001 From: Vandenn Date: Thu, 2 May 2019 23:59:01 +0800 Subject: [PATCH 0426/1853] doc: update configure_logging docs to discourage use with CrawlerProcess --- docs/topics/logging.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 8e280d929..dea0528db 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -238,9 +238,10 @@ scrapy.utils.log module .. autofunction:: configure_logging - ``configure_logging`` is automatically called when using Scrapy commands, - but needs to be called explicitly when running custom scripts. In that - case, its usage is not required but it's recommended. + ``configure_logging`` is automatically called when using Scrapy commands + or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly + when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`. + In that case, its usage is not required but it's recommended. If you plan on configuring the handlers yourself is still recommended you call this function, passing ``install_root_handler=False``. Bear in mind From 8bd207a2f639216eb51f61ae312dfd22f4b39781 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Sun, 28 Apr 2019 21:47:47 +0200 Subject: [PATCH 0427/1853] Add documentation --- docs/topics/contracts.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 70f20d4ed..3aa32cba8 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -120,3 +120,22 @@ get the failures pretty printed:: for header in self.args: if header not in response.headers: raise ContractFail('X-CustomHeader not present') + + +Detecting check run +=================== +It is not encouraged to created different behaviour when running test. +However, sometimes it is useful to know when a spider is started by scrapy check. +It can for example be needed to enforce less settings to be set, or to disable some +uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment +variable is set. This can be retrieved via ``os.environ``:: + + import os + import scrapy + + class ExampleSpider(scrapy.Spider): + name = 'example' + + def __init__(self): + if os.environ.get('SCRAPY_CHECK'): + # Do some scraper adjustments when check is running \ No newline at end of file From f6485e669772a940c3c319c71dbcca7bd747d57a Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Fri, 3 May 2019 13:53:45 +0200 Subject: [PATCH 0428/1853] Restore alphabetic order and two lines between import and code --- scrapy/utils/misc.py | 1 + tests/test_utils_misc/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b2164d4a8..f638adb25 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -12,6 +12,7 @@ from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem + _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index eeb995e48..e109d5343 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,12 +3,13 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules from tests import mock __doctests__ = ['scrapy.utils.misc'] + class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): From bc1a92921364de8f4616feb0ea7dcb7b6d42d2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 3 May 2019 14:42:12 +0200 Subject: [PATCH 0429/1853] Improve the documentation about detecting check runs --- docs/topics/contracts.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 3aa32cba8..9337375bb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -122,13 +122,12 @@ get the failures pretty printed:: raise ContractFail('X-CustomHeader not present') -Detecting check run -=================== -It is not encouraged to created different behaviour when running test. -However, sometimes it is useful to know when a spider is started by scrapy check. -It can for example be needed to enforce less settings to be set, or to disable some -uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment -variable is set. This can be retrieved via ``os.environ``:: +Detecting check runs +==================== + +When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is +set to the ``true`` string. You can use `os.environ`_ to perform any change to +your spiders or your settings when ``scrapy check`` is used:: import os import scrapy @@ -138,4 +137,6 @@ variable is set. This can be retrieved via ``os.environ``:: def __init__(self): if os.environ.get('SCRAPY_CHECK'): - # Do some scraper adjustments when check is running \ No newline at end of file + pass # Do some scraper adjustments when a check is running + +.. _os.environ: https://docs.python.org/3/library/os.html#os.environ From 5814344adfc315a63f43237f02ed75db52765b7d Mon Sep 17 00:00:00 2001 From: Jeffallan <23423962+Jeffallan@users.noreply.github.com> Date: Sat, 4 May 2019 14:15:47 -0500 Subject: [PATCH 0430/1853] Update telnetconsole.rst Change spelling of bellow to below. --- docs/topics/telnetconsole.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index bf2ffa443..1eb705f05 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -45,7 +45,7 @@ the console you need to type:: >>> By default Username is ``scrapy`` and Password is autogenerated. The -autogenerated Password can be seen on scrapy logs like the example bellow:: +autogenerated Password can be seen on scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 From 3a7850fa158148e6c6096add09b555e46949bd51 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 5 May 2019 18:45:40 +0530 Subject: [PATCH 0431/1853] Update contributing.rst --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2fbe30a00..51b5da59e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -262,7 +262,7 @@ And their unit-tests are in:: .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _pull request: https://help.github.com/send-pull-requests/ +.. _pull request: https://help.github.com/en/articles/creating-a-pull-request .. _pytest: https://docs.pytest.org/en/latest/usage.html .. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html .. _tox: https://pypi.python.org/pypi/tox From 611249bb7f3a7bb5a92a67d90d6c97f17494768f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 8 May 2019 12:52:29 +0200 Subject: [PATCH 0432/1853] Implement the METAREFRESH_IGNORE_TAGS setting --- docs/topics/downloader-middleware.rst | 10 ++++++++++ scrapy/downloadermiddlewares/redirect.py | 4 +++- scrapy/settings/default_settings.py | 1 + scrapy/utils/response.py | 4 ++-- tests/test_downloadermiddleware_redirect.py | 19 +++++++++++++++++++ 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f2f3ef466..fa65f66ed 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -805,6 +805,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` +* :setting:`METAREFRESH_IGNORE_TAGS` * :setting:`METAREFRESH_MAXDELAY` This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, @@ -826,6 +827,15 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. +.. setting:: METAREFRESH_IGNORE_TAGS + +METAREFRESH_IGNORE_TAGS +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``['script', 'noscript']`` + +Meta tags within these tags are ignored. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index cb59d3fd2..49468a2e4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -88,6 +88,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings): super(MetaRefreshMiddleware, self).__init__(settings) + self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY', settings.getint('METAREFRESH_MAXDELAY')) @@ -96,7 +97,8 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): not isinstance(response, HtmlResponse): return response - interval, url = get_meta_refresh(response) + interval, url = get_meta_refresh(response, + ignore_tags=self._ignore_tags) if url and interval < self._maxdelay: redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, 'meta refresh') diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9986827d8..1ce1516e5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -221,6 +221,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True +METAREFRESH_IGNORE_TAGS = ['script', 'noscript'] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = '' diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index bf276b5ca..122af28b0 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -31,12 +31,12 @@ def get_base_url(response): _metaref_cache = weakref.WeakKeyDictionary() -def get_meta_refresh(response): +def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] _metaref_cache[response] = html.get_meta_refresh(text, response.url, - response.encoding, ignore_tags=('script', 'noscript')) + response.encoding, ignore_tags=ignore_tags) return _metaref_cache[response] diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 6c81c94ca..0e841489d 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -279,5 +279,24 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh']) self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh']) + def test_ignore_tags_default(self): + req = Request(url='http://example.org') + body = ('''''') + rsp = HtmlResponse(req.url, body=body.encode()) + response = self.mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) + + def test_ignore_tags_empty_list(self): + crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []}) + mw = MetaRefreshMiddleware.from_crawler(crawler) + req = Request(url='http://example.org') + body = ('''''') + rsp = HtmlResponse(req.url, body=body.encode()) + req2 = mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, 'http://example.org/newpage') + if __name__ == "__main__": unittest.main() From e667ca76820a53ac3abf34604fc284761f936bb9 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Fri, 24 May 2019 21:45:53 +0900 Subject: [PATCH 0433/1853] Account for mangling when serializing requests with private callbacks --- scrapy/utils/reqser.py | 6 +++++- tests/test_utils_reqser.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 959dddbd5..8c99763cf 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -75,7 +75,11 @@ def _find_method(obj, func): pass else: if func_self is obj: - return six.get_method_function(func).__name__ + name = six.get_method_function(func).__name__ + if name.startswith('__'): + classname = obj.__class__.__name__.lstrip('_') + name = '_%s%s' % (classname, name) + return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index dcc070b8f..f7191fcef 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -68,6 +68,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider._TestSpider__parse_item_private, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -87,6 +93,9 @@ class TestSpider(Spider): def handle_error(self, failure): pass + def __parse_item_private(self, response): + pass + class CustomRequest(Request): pass From 7d36fa7435d2147c7dfd6a87733187823431b61c Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Fri, 24 May 2019 10:32:55 -0300 Subject: [PATCH 0434/1853] Fix documentation for spiderloader --- docs/topics/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index ba832ab5d..e1623287d 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -154,7 +154,7 @@ Settings API SpiderLoader API ================ -.. module:: scrapy.loader +.. module:: scrapy.spiderloader :synopsis: The spider loader .. class:: SpiderLoader From 0ee2284fcc23ccb2b8a4da8fb561a232cc328fe0 Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Fri, 24 May 2019 11:11:15 -0300 Subject: [PATCH 0435/1853] Add 429 to RETRY_HTTP_CODES --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9986827d8..2afa7b321 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -238,7 +238,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False From 144afcee7973ab97d6c8d89fec007046cc878e3d Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Sat, 25 May 2019 00:52:00 +0900 Subject: [PATCH 0436/1853] Use regex to check for private methods --- scrapy/utils/reqser.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 8c99763cf..07c51aaff 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -2,12 +2,16 @@ Helper functions for serializing (and deserializing) requests. """ import six +import re from scrapy.http import Request from scrapy.utils.python import to_unicode, to_native_str from scrapy.utils.misc import load_object +private_name_regex = re.compile('^__[^_](.*[^_])?_?$') + + def request_to_dict(request, spider=None): """Convert Request object to a dict. @@ -76,7 +80,7 @@ def _find_method(obj, func): else: if func_self is obj: name = six.get_method_function(func).__name__ - if name.startswith('__'): + if private_name_regex.search(name): classname = obj.__class__.__name__.lstrip('_') name = '_%s%s' % (classname, name) return name From 461682fc3dca72d9a34ddc22ad1896787c9dc518 Mon Sep 17 00:00:00 2001 From: Claudio Salazar Date: Sat, 25 May 2019 11:01:19 +0200 Subject: [PATCH 0437/1853] Whitelist form methods in FormRequest.from_response method --- scrapy/http/request/form.py | 7 ++++++- tests/test_http_request.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c2413b431..2182b9b53 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -18,6 +18,7 @@ from scrapy.utils.response import get_base_url class FormRequest(Request): + valid_form_methods = ['GET', 'POST', 'DIALOG'] def __init__(self, *args, **kwargs): formdata = kwargs.pop('formdata', None) @@ -48,7 +49,11 @@ class FormRequest(Request): form = _get_form(response, formname, formid, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = _get_form_url(form, kwargs.pop('url', None)) - method = kwargs.pop('method', form.method) + + method = kwargs.pop('method', form.method).upper() + if method not in cls.valid_form_methods: + raise ValueError('Invalid form method in chosen form') + return cls(url=url, method=method, formdata=formdata, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 49f148016..8fdafb286 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1100,6 +1100,19 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formcss="input[name='abc']") + def test_from_response_valid_form_methods(self): + body = """ + + """ + + for method in self.request_class.valid_form_methods: + response = _buildresponse(body % method) + r1 = self.request_class.from_response(response) + self.assertEqual(r1.method, method) + + response = _buildresponse(body % 'UNKNOWN') + self.assertRaises(ValueError, self.request_class.from_response, response) + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) From da82ede8a0751cf3e8496f35252eb0dcef4f197e Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 25 May 2019 17:19:10 +0530 Subject: [PATCH 0438/1853] describe method as a command --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dfbcdb8fa..236150059 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -526,7 +526,7 @@ defines the methods described below. .. method:: retrieve_response(spider, request) - Returns response if present in cache, or ``None`` otherwise. + Return response if present in cache, or ``None`` otherwise. :param spider: the spider which generated the request :type spider: :class:`~scrapy.spiders.Spider` object @@ -536,7 +536,7 @@ defines the methods described below. .. method:: store_response(spider, request, response) - Stores the given response in the cache. + Store the given response in the cache. :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object From 18f01ea6ecf1dba77b25d8d1f62c80ed0f9a13f5 Mon Sep 17 00:00:00 2001 From: mar-heaven <775650117@qq.com> Date: Mon, 27 May 2019 17:15:30 +0800 Subject: [PATCH 0439/1853] remove a "is" When I translated in Chinese, I found a needless "is" --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8c4049f85..79eecfc3e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -657,7 +657,7 @@ SitemapSpider .. attribute:: sitemap_follow - A list of regexes of sitemap that should be followed. This is is only + A list of regexes of sitemap that should be followed. This is only for sites that use `Sitemap index files`_ that point to other sitemap files. From 72b7d3e90ac2d21ffdd0c44878ec1a5a5d0fa5ce Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Mon, 27 May 2019 23:30:23 +0900 Subject: [PATCH 0440/1853] Make the regex align to the spec better; add unit tests for name variations --- scrapy/utils/reqser.py | 2 +- tests/test_utils_reqser.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 07c51aaff..04665a2d4 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -9,7 +9,7 @@ from scrapy.utils.python import to_unicode, to_native_str from scrapy.utils.misc import load_object -private_name_regex = re.compile('^__[^_](.*[^_])?_?$') +private_name_regex = re.compile('^__.*[^_]_?$') def request_to_dict(request, spider=None): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index f7191fcef..b49450ac5 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -3,7 +3,7 @@ import unittest from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex class RequestSerializationTest(unittest.TestCase): @@ -74,6 +74,28 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_name_matching(self): + self.assertTrue(private_name_regex.search('__a')) + self.assertTrue(private_name_regex.search('__a_')) + self.assertTrue(private_name_regex.search('__a_a')) + self.assertTrue(private_name_regex.search('__a_a_')) + self.assertTrue(private_name_regex.search('__a__a')) + self.assertTrue(private_name_regex.search('__a__a_')) + self.assertTrue(private_name_regex.search('__a___a')) + self.assertTrue(private_name_regex.search('__a___a_')) + self.assertTrue(private_name_regex.search('___a')) + self.assertTrue(private_name_regex.search('___a_')) + self.assertTrue(private_name_regex.search('___a_a')) + self.assertTrue(private_name_regex.search('___a_a_')) + self.assertTrue(private_name_regex.search('____a_a_')) + + self.assertFalse(private_name_regex.search('_a')) + self.assertFalse(private_name_regex.search('_a_')) + self.assertFalse(private_name_regex.search('__a__')) + self.assertFalse(private_name_regex.search('__')) + self.assertFalse(private_name_regex.search('___')) + self.assertFalse(private_name_regex.search('____')) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) From 9af91a26b035a10e9303227ad9ddd5e043725514 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Tue, 28 May 2019 01:40:26 +0900 Subject: [PATCH 0441/1853] Replace regex usage --- scrapy/utils/reqser.py | 10 +++++----- tests/test_utils_reqser.py | 40 +++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 04665a2d4..40223661f 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -2,16 +2,12 @@ Helper functions for serializing (and deserializing) requests. """ import six -import re from scrapy.http import Request from scrapy.utils.python import to_unicode, to_native_str from scrapy.utils.misc import load_object -private_name_regex = re.compile('^__.*[^_]_?$') - - def request_to_dict(request, spider=None): """Convert Request object to a dict. @@ -71,6 +67,10 @@ def request_from_dict(d, spider=None): flags=d.get('flags')) +def _is_private_method(name): + return name.startswith('__') and not name.endswith('__') + + def _find_method(obj, func): if obj: try: @@ -80,7 +80,7 @@ def _find_method(obj, func): else: if func_self is obj: name = six.get_method_function(func).__name__ - if private_name_regex.search(name): + if _is_private_method(name): classname = obj.__class__.__name__.lstrip('_') name = '_%s%s' % (classname, name) return name diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index b49450ac5..fad5b6003 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -3,7 +3,7 @@ import unittest from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex +from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method class RequestSerializationTest(unittest.TestCase): @@ -75,26 +75,26 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r, spider=self.spider) def test_private_callback_name_matching(self): - self.assertTrue(private_name_regex.search('__a')) - self.assertTrue(private_name_regex.search('__a_')) - self.assertTrue(private_name_regex.search('__a_a')) - self.assertTrue(private_name_regex.search('__a_a_')) - self.assertTrue(private_name_regex.search('__a__a')) - self.assertTrue(private_name_regex.search('__a__a_')) - self.assertTrue(private_name_regex.search('__a___a')) - self.assertTrue(private_name_regex.search('__a___a_')) - self.assertTrue(private_name_regex.search('___a')) - self.assertTrue(private_name_regex.search('___a_')) - self.assertTrue(private_name_regex.search('___a_a')) - self.assertTrue(private_name_regex.search('___a_a_')) - self.assertTrue(private_name_regex.search('____a_a_')) + self.assertTrue(_is_private_method('__a')) + self.assertTrue(_is_private_method('__a_')) + self.assertTrue(_is_private_method('__a_a')) + self.assertTrue(_is_private_method('__a_a_')) + self.assertTrue(_is_private_method('__a__a')) + self.assertTrue(_is_private_method('__a__a_')) + self.assertTrue(_is_private_method('__a___a')) + self.assertTrue(_is_private_method('__a___a_')) + self.assertTrue(_is_private_method('___a')) + self.assertTrue(_is_private_method('___a_')) + self.assertTrue(_is_private_method('___a_a')) + self.assertTrue(_is_private_method('___a_a_')) + self.assertTrue(_is_private_method('____a_a_')) - self.assertFalse(private_name_regex.search('_a')) - self.assertFalse(private_name_regex.search('_a_')) - self.assertFalse(private_name_regex.search('__a__')) - self.assertFalse(private_name_regex.search('__')) - self.assertFalse(private_name_regex.search('___')) - self.assertFalse(private_name_regex.search('____')) + self.assertFalse(_is_private_method('_a')) + self.assertFalse(_is_private_method('_a_')) + self.assertFalse(_is_private_method('__a__')) + self.assertFalse(_is_private_method('__')) + self.assertFalse(_is_private_method('___')) + self.assertFalse(_is_private_method('____')) def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) From bcad8947e8192448ab3bd59489444efb567f8793 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Mon, 3 Jun 2019 20:41:02 +0900 Subject: [PATCH 0442/1853] Support inherited private method names --- scrapy/utils/reqser.py | 9 +++++++-- tests/test_utils_reqser.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 40223661f..d1f472e6e 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -81,8 +81,13 @@ def _find_method(obj, func): if func_self is obj: name = six.get_method_function(func).__name__ if _is_private_method(name): - classname = obj.__class__.__name__.lstrip('_') - name = '_%s%s' % (classname, name) + qualname = getattr(func, '__qualname__', None) + if qualname is None: + classname = obj.__class__.__name__.lstrip('_') + name = '_%s%s' % (classname, name) + else: + splits = qualname.split('.') + name = '_%s%s' % (splits[-2], splits[-1]) return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index fad5b6003..31577bc8c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import unittest +import sys from scrapy.http import Request, FormRequest from scrapy.spiders import Spider @@ -74,6 +75,14 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_mixin_private_callback_serialization(self): + if sys.version_info[0] < 3: + return + r = Request("http://www.example.com", + callback=self.spider._TestSpiderMixin__mixin_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_name_matching(self): self.assertTrue(_is_private_method('__a')) self.assertTrue(_is_private_method('__a_')) @@ -106,7 +115,12 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpider(Spider): +class TestSpiderMixin(object): + def __mixin_callback(self, response): + pass + + +class TestSpider(Spider, TestSpiderMixin): name = 'test' def parse_item(self, response): From ea209a0ea7815f68a04a5ccab79a2b4f4a146647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Jun 2019 19:21:40 +0200 Subject: [PATCH 0443/1853] Fix module double indexing issues in the documentation --- docs/topics/stats.rst | 3 +-- docs/topics/telnetconsole.rst | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index dd0c6216b..38648ec55 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The default Stats Collector used is the :class:`MemoryStatsCollector`. -.. module:: scrapy.statscollectors - :synopsis: Stats Collectors +.. currentmodule:: scrapy.statscollectors MemoryStatsCollector -------------------- diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 1eb705f05..7db7e4f6b 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -1,12 +1,11 @@ +.. currentmodule:: scrapy.extensions.telnet + .. _topics-telnetconsole: ============== Telnet Console ============== -.. module:: scrapy.extensions.telnet - :synopsis: The Telnet Console - Scrapy comes with a built-in telnet console for inspecting and controlling a Scrapy running process. The telnet console is just a regular python shell running inside the Scrapy process, so you can do literally anything from it. From c7b5ad0e20dc736a8a08b62134c9418439fb7077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 4 Jan 2019 18:17:35 +0100 Subject: [PATCH 0444/1853] Add a Sphinx extension to generate documentation coverage information --- docs/Makefile | 3 +++ docs/conf.py | 9 ++++++++- docs/contributing.rst | 9 +++++++++ docs/requirements.txt | 2 +- tox.ini | 6 ++++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 187f03c4c..ff68bf1ae 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -82,6 +82,9 @@ pydoc-topics: build @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ "into the Lib/ directory" +coverage: BUILDER = coverage +coverage: build + htmlview: html $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ os.path.realpath('build/html/index.html'))" diff --git a/docs/conf.py b/docs/conf.py index a54a6bbe9..832626f6b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,8 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'scrapydocs', - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', ] # Add any paths that contain templates here, relative to this directory. @@ -218,3 +219,9 @@ linkcheck_ignore = [ 'http://localhost:\d+', 'http://hg.scrapy.org', 'http://directory.google.com/' ] + + +# Options for the Coverage extension +# ---------------------------------- +coverage_ignore_pyobjects = [ +] diff --git a/docs/contributing.rst b/docs/contributing.rst index 51b5da59e..b4f91ea8d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -99,6 +99,15 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +* if you're adding a private API, please add a regular expression to the + ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new + private API from documentation coverage checks. + + To see if your private API is skipped properly, generate a documentation + coverage report as follows:: + + tox -e docs-coverage + .. _submitting-patches: Submitting patches diff --git a/docs/requirements.txt b/docs/requirements.txt index 8e7611d21..379da9994 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.6 +Sphinx>=2.1 sphinx_rtd_theme \ No newline at end of file diff --git a/tox.ini b/tox.ini index 0c0f8f7b7..157a8b3ed 100644 --- a/tox.ini +++ b/tox.ini @@ -105,6 +105,12 @@ deps = {[docs]deps} commands = sphinx-build -W -b html . {envtmpdir}/html +[testenv:docs-coverage] +changedir = {[docs]changedir} +deps = {[docs]deps} +commands = + sphinx-build -b coverage . {envtmpdir}/coverage + [testenv:docs-links] changedir = {[docs]changedir} deps = {[docs]deps} From c81e15ed6ede552c499ae3ac4e03af27b1f9ed89 Mon Sep 17 00:00:00 2001 From: Artem Kuchumov Date: Wed, 5 Jun 2019 13:15:23 +0500 Subject: [PATCH 0445/1853] Tutorial: scrapy shell example should say "text" not "title" (#3807) Tutorial: scrapy shell example should say "text" not "title" --- docs/intro/tutorial.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8bd2d27dd..a190ce407 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -379,11 +379,11 @@ variable, so that we can run our CSS selectors directly on a particular quote:: >>> quote = response.css("div.quote")[0] -Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +Now, let's extract ``text``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").get() - >>> title + >>> text = quote.css("span.text::text").get() + >>> text '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' >>> author = quote.css("small.author::text").get() >>> author From 9c81721c407ff41ef9dce2c33e26ac477355cf1f Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Wed, 5 Jun 2019 23:43:56 +0900 Subject: [PATCH 0446/1853] Add tests for private method name mangling --- scrapy/utils/reqser.py | 18 +++++++++++------- tests/test_utils_reqser.py | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d1f472e6e..3c463cfed 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,6 +71,16 @@ def _is_private_method(name): return name.startswith('__') and not name.endswith('__') +def _mangle_private_name(obj, func, name): + qualname = getattr(func, '__qualname__', None) + if qualname is None: + classname = obj.__class__.__name__.lstrip('_') + return '_%s%s' % (classname, name) + else: + splits = qualname.split('.') + return '_%s%s' % (splits[-2], splits[-1]) + + def _find_method(obj, func): if obj: try: @@ -81,13 +91,7 @@ def _find_method(obj, func): if func_self is obj: name = six.get_method_function(func).__name__ if _is_private_method(name): - qualname = getattr(func, '__qualname__', None) - if qualname is None: - classname = obj.__class__.__name__.lstrip('_') - name = '_%s%s' % (classname, name) - else: - splits = qualname.split('.') - name = '_%s%s' % (splits[-2], splits[-1]) + return _mangle_private_name(obj, func, name) return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 31577bc8c..7f9e31daa 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -2,9 +2,11 @@ import unittest import sys +import six + from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method +from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name class RequestSerializationTest(unittest.TestCase): @@ -105,6 +107,18 @@ class RequestSerializationTest(unittest.TestCase): self.assertFalse(_is_private_method('___')) self.assertFalse(_is_private_method('____')) + def _assert_mangles_to(self, obj, name): + self.assertEqual( + _mangle_private_name(obj, getattr(obj, name), name), + name + ) + + def test_private_name_mangling(self): + self._assert_mangles_to( + self.spider, '_TestSpider__parse_item_private') + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) From 3dd3e8c29863683d60f9c4f74aacac3103703061 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Wed, 5 Jun 2019 23:49:54 +0900 Subject: [PATCH 0447/1853] Restrict different class mangling tests to Py 3+ --- tests/test_utils_reqser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 7f9e31daa..57dc5db53 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -116,8 +116,9 @@ class RequestSerializationTest(unittest.TestCase): def test_private_name_mangling(self): self._assert_mangles_to( self.spider, '_TestSpider__parse_item_private') - self._assert_mangles_to( - self.spider, '_TestSpiderMixin__mixin_callback') + if sys.version_info[0] >= 3: + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) From 6af1dc89aa5988ebbfbef90afdafa84736f3993c Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Thu, 6 Jun 2019 04:25:19 +0900 Subject: [PATCH 0448/1853] Fix mangling test --- tests/test_utils_reqser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 57dc5db53..e5a09dcf1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -108,8 +108,9 @@ class RequestSerializationTest(unittest.TestCase): self.assertFalse(_is_private_method('____')) def _assert_mangles_to(self, obj, name): + func = getattr(obj, name) self.assertEqual( - _mangle_private_name(obj, getattr(obj, name), name), + _mangle_private_name(obj, func, func.__name__), name ) From 0c50879568dee2363df5cbe25e9bdd7adaed5da4 Mon Sep 17 00:00:00 2001 From: Claudio Salazar Date: Thu, 6 Jun 2019 22:10:59 +0200 Subject: [PATCH 0449/1853] Change behavior to use method GET when there are unknown methods in the form --- scrapy/http/request/form.py | 2 +- tests/test_http_request.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2182b9b53..8b29aae4b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -52,7 +52,7 @@ class FormRequest(Request): method = kwargs.pop('method', form.method).upper() if method not in cls.valid_form_methods: - raise ValueError('Invalid form method in chosen form') + method = 'GET' return cls(url=url, method=method, formdata=formdata, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 8fdafb286..258b48dce 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1107,11 +1107,12 @@ class FormRequestTest(RequestTest): for method in self.request_class.valid_form_methods: response = _buildresponse(body % method) - r1 = self.request_class.from_response(response) - self.assertEqual(r1.method, method) + r = self.request_class.from_response(response) + self.assertEqual(r.method, method) response = _buildresponse(body % 'UNKNOWN') - self.assertRaises(ValueError, self.request_class.from_response, response) + r = self.request_class.from_response(response) + self.assertEqual(r.method, 'GET') def _buildresponse(body, **kwargs): From bd8a10384b462dd56b33668e8b92e4a148fd6fba Mon Sep 17 00:00:00 2001 From: Sortafreel Date: Fri, 7 Jun 2019 01:50:03 +0300 Subject: [PATCH 0450/1853] Add values (if there're any) when initiating items from dicts https://github.com/scrapy/scrapy/issues/3804 --- scrapy/loader/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index a7c75a46a..295a8e42d 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -35,6 +35,8 @@ class ItemLoader(object): self.parent = parent self._local_item = context['item'] = item self._local_values = defaultdict(list) + for field_name, value in item.items(): + self.add_value(field_name, value) @property def _values(self): From 754f52b02781097c8ca6835e057815c7653062d4 Mon Sep 17 00:00:00 2001 From: Sortafreel Date: Fri, 7 Jun 2019 03:20:45 +0300 Subject: [PATCH 0451/1853] Preprocess values if item built from dict. https://github.com/scrapy/scrapy/issues/3804 --- scrapy/loader/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 295a8e42d..7c7f66866 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -35,8 +35,9 @@ class ItemLoader(object): self.parent = parent self._local_item = context['item'] = item self._local_values = defaultdict(list) + # Preprocess values if item built from dict for field_name, value in item.items(): - self.add_value(field_name, value) + self._values[field_name] = self._process_input_value(field_name, value) @property def _values(self): From c7ba72b5dc9da3435eb1ec303b991d05ba40ba1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Jun 2019 17:10:14 +0200 Subject: [PATCH 0452/1853] Skip scrapy.contracts private APIs in the documentation coverage report --- docs/conf.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 832626f6b..bf222b361 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -224,4 +224,17 @@ linkcheck_ignore = [ # Options for the Coverage extension # ---------------------------------- coverage_ignore_pyobjects = [ + # Contract’s add_pre_hook and add_post_hook are not documented because + # they should be transparent to contract developers, for whom pre_hook and + # post_hook should be the actual concern. + r'\bContract\.add_(pre|post)_hook$', + + # ContractsManager is an internal class, developers are not expected to + # interact with it directly in any way. + r'\bContractsManager\b$', + + # For default contracts we only want to document their general purpose in + # their constructor, the methods they reimplement to achieve that purpose + # should be irrelevant to developers using those contracts. + r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', ] From a1bca6a8e722af53241e51bbf758e7bd67671801 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 07:36:29 +0300 Subject: [PATCH 0453/1853] Add tests. --- scrapy/loader/__init__.py | 1 + tests/test_loader.py | 65 ++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 7c7f66866..20f0f90c3 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -36,6 +36,7 @@ class ItemLoader(object): self._local_item = context['item'] = item self._local_values = defaultdict(list) # Preprocess values if item built from dict + # Values need to be added to item._values if added them from dict (not with add_values) for field_name, value in item.items(): self._values[field_name] = self._process_input_value(field_name, value) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8b58e4dbd..eb4a01572 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -419,6 +419,29 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item['url'], u'rabbit.hole') self.assertEqual(item['summary'], u'rabbithole') + def test_create_item_from_dict(self): + class TestItem(Item): + title = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + + input_item = {'title': 'Test item title 1'} + il = TestItemLoader(item=input_item) + # Getting output value mustn't remove value from item + self.assertEqual(il.load_item(), { + 'title': 'Test item title 1', + }) + self.assertEqual(il.get_output_value('title'), 'Test item title 1') + self.assertEqual(il.load_item(), { + 'title': 'Test item title 1', + }) + + input_item = {'title': 'Test item title 2'} + il = TestItemLoader(item=input_item) + # Values from dict must be added to item _values + self.assertEqual(il._values.get('title'), 'Test item title 2') + class ProcessorsTest(unittest.TestCase): @@ -709,28 +732,28 @@ class SubselectorLoaderTest(unittest.TestCase): class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) + def test_output(self): + for l in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[l] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(l, test, expected) + ) if __name__ == "__main__": From 7dad2f7b130c426f2a8aee320ccbc378752a9568 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 07:43:03 +0300 Subject: [PATCH 0454/1853] Add more tests. --- tests/test_loader.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_loader.py b/tests/test_loader.py index eb4a01572..241630ab3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -442,6 +442,20 @@ class BasicItemLoaderTest(unittest.TestCase): # Values from dict must be added to item _values self.assertEqual(il._values.get('title'), 'Test item title 2') + input_item = {'title': [u'Test item title 3', u'Test item 4']} + il = TestItemLoader(item=input_item) + # Same rules must work for lists + self.assertEqual(il._values.get('title'), + [u'Test item title 3', u'Test item 4']) + self.assertEqual(il.load_item(), { + 'title': [u'Test item title 3', u'Test item 4'], + }) + self.assertEqual(il.get_output_value('title'), + [u'Test item title 3', u'Test item 4']) + self.assertEqual(il.load_item(), { + 'title': [u'Test item title 3', u'Test item 4'], + }) + class ProcessorsTest(unittest.TestCase): From 0da972339bb174156b08a3ae34ece7fddea1e48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Jun 2019 14:11:38 +0200 Subject: [PATCH 0455/1853] Require Twisted<=19.2.0 for Python 3.4 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd666e93c..4dc6d18c1 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,8 @@ setup( ], python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', install_requires=[ - 'Twisted>=13.1.0', + 'Twisted>=13.1.0;python_version!="3.4"', + 'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"', 'w3lib>=1.17.0', 'queuelib', 'lxml', From fe0f80f2f422d4047a8b6230d66eb853d443d90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Jun 2019 15:50:41 +0200 Subject: [PATCH 0456/1853] Set the cloned directory as PYTHONPATH in appveyor.yml --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 93cfd469e..7fd636864 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,7 +12,8 @@ branches: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" + - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - "pip install -U tox" build: false From cdeccac6d6ccd0034a5f007ed371c1d481b32c26 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 17:38:06 +0300 Subject: [PATCH 0457/1853] Linting (return previous indentation). --- tests/test_loader.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_loader.py b/tests/test_loader.py index 241630ab3..5a8ee1b2e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -746,28 +746,28 @@ class SubselectorLoaderTest(unittest.TestCase): class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) + def test_output(self): + for l in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[l] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(l, test, expected) + ) if __name__ == "__main__": From b8900ec6a698cb4e27424de57ca5593f1c7300e7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 17 Jun 2019 00:06:44 +0530 Subject: [PATCH 0458/1853] removes unused var --- tests/test_downloadermiddleware.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 0f420b70d..03564e748 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -123,7 +123,6 @@ class ProcessRequestInvalidOutput(ManagerTestCase): def test_invalid_process_request(self): req = Request('http://example.com/index.html') - resp = Response('http://example.com/index.html') class InvalidProcessRequestMiddleware: def process_request(self, request, spider): @@ -143,7 +142,6 @@ class ProcessResponseInvalidOutput(ManagerTestCase): def test_invalid_process_response(self): req = Request('http://example.com/index.html') - resp = Response('http://example.com/index.html') class InvalidProcessResponseMiddleware: def process_response(self, request, response, spider): @@ -163,7 +161,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): def test_invalid_process_exception(self): req = Request('http://example.com/index.html') - resp = Response('http://example.com/index.html') class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): From 8d1e0e09bb6fdeb4f1348b408a268c92dc9e7a8f Mon Sep 17 00:00:00 2001 From: Mabel Villalba Date: Thu, 20 Jun 2019 10:06:06 +0200 Subject: [PATCH 0459/1853] [itemloader-errors] added error message in get_value --- scrapy/loader/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 20f0f90c3..5055de015 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -106,11 +106,17 @@ class ItemLoader(object): value = arg_to_iter(value) value = flatten(extract_regex(regex, x) for x in value) - for proc in processors: + for _proc in processors: if value is None: break - proc = wrap_loader_context(proc, self.context) - value = proc(value) + proc = wrap_loader_context(_proc, self.context) + try: + value = proc(value) + except Exception as e: + raise ValueError( + "Error with processor %s value=%r error='%s: %s'" % + (_proc.__class__.__name__, value, type(e).__name__, + str(e))) return value def load_item(self): From 663352b2a5250c377bbbe2d53c5d5b7da3a1836a Mon Sep 17 00:00:00 2001 From: Mabel Villalba Date: Thu, 20 Jun 2019 10:10:16 +0200 Subject: [PATCH 0460/1853] [itemloader-errors] added error message to _process_input_value --- scrapy/loader/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 5055de015..fc8b10bee 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -155,9 +155,15 @@ class ItemLoader(object): return proc def _process_input_value(self, field_name, value): - proc = self.get_input_processor(field_name) - proc = wrap_loader_context(proc, self.context) - return proc(value) + _proc = self.get_input_processor(field_name) + proc = wrap_loader_context(_proc, self.context) + try: + return proc(value) + except Exception as e: + raise ValueError( + "Error with inputput processor %s: field=%r value=%r " + "error='%s: %s'" % (_proc.__class__.__name__, field_name, + value, type(e).__name__, str(e))) def _get_item_field_attr(self, field_name, key, default=None): if isinstance(self.item, Item): From f4f2b1695c4d7bc69e5cb19c33a3a47f69bd1e8d Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Mon, 24 Jun 2019 07:38:05 -0300 Subject: [PATCH 0461/1853] Fix a memory leak on the Media Pipeline (Files and Images) (#3813) We're storing exceptions captured by Twisted on the media pipeline cache, but we're also using the defer.returnValue method with our own methods decorated with @defer.inlineCallbacks. The defer.returnValue method passes returned values forward by throwing a defer._DefGen_Return exception, which in its turn extends the BaseException class and is captured by Twisted. This way, the latest exception stored in the Failure's object may also have an HtmlResponse object in its __context__ attribute. As the Response object also keeps track of the Request object that has originated it, you could figure it out how many RAM we're wasting here. This could easily lead to a Memory Leak problem when running spiders with Media Pipeline enabled and a particular Request set that tends to raise a significant number of exceptions. Example triggers: - media requests with 404 status responses - user land exceptins coming from custom middlewares - etc. --- scrapy/pipelines/media.py | 26 +++++++++++- tests/test_pipeline_media.py | 77 +++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 404bbf5bf..95dca9a3f 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -3,7 +3,7 @@ from __future__ import print_function import functools import logging from collections import defaultdict -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return from twisted.python.failure import Failure from scrapy.settings import Settings @@ -139,6 +139,30 @@ class MediaPipeline(object): result.cleanFailure() result.frames = [] result.stack = None + + # This code fixes a memory leak by avoiding to keep references to + # the Request and Response objects on the Media Pipeline cache. + # + # Twisted inline callbacks pass return values using the function + # twisted.internet.defer.returnValue, which encapsulates the return + # value inside a _DefGen_Return base exception. + # + # What happens when the media_downloaded callback raises another + # exception, for example a FileException('download-error') when + # the Response status code is not 200 OK, is that it stores the + # _DefGen_Return exception on the FileException context. + # + # To avoid keeping references to the Response and therefore Request + # objects on the Media Pipeline cache, we should wipe the context of + # the exception encapsulated by the Twisted Failure when its a + # _DefGen_Return instance. + # + # This problem does not occur in Python 2.7 since we don't have + # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). + context = getattr(result.value, '__context__', None) + if isinstance(context, _DefGen_Return): + setattr(result.value, '__context__', None) + info.downloading.remove(fp) info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 5f6a6d9e6..28e39cefa 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,15 +1,19 @@ from __future__ import print_function + +import sys + from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure from twisted.internet import reactor -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks, returnValue from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.request import request_fingerprint from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy import signals @@ -90,6 +94,77 @@ class BaseMediaPipelineTestCase(unittest.TestCase): self.pipe._modify_media_request(request) assert request.meta == {'handle_httpstatus_all': True} + def test_should_remove_req_res_references_before_caching_the_results(self): + """Regression test case to prevent a memory leak in the Media Pipeline. + + The memory leak is triggered when an exception is raised when a Response + scheduled by the Media Pipeline is being returned. For example, when a + FileException('download-error') is raised because the Response status + code is not 200 OK. + + It happens because we are keeping a reference to the Response object + inside the FileException context. This is caused by the way Twisted + return values from inline callbacks. It raises a custom exception + encapsulating the original return value. + + The solution is to remove the exception context when this context is a + _DefGen_Return instance, the BaseException used by Twisted to pass the + returned value from those inline callbacks. + + Maybe there's a better and more reliable way to test the case described + here, but it would be more complicated and involve running - or at least + mocking - some async steps from the Media Pipeline. The current test + case is simple and detects the problem very fast. On the other hand, it + would not detect another kind of leak happening due to old object + references being kept inside the Media Pipeline cache. + + This problem does not occur in Python 2.7 since we don't have Exception + Chaining (https://www.python.org/dev/peps/pep-3134/). + """ + # Create sample pair of Request and Response objects + request = Request('http://url') + response = Response('http://url', body=b'', request=request) + + # Simulate the Media Pipeline behavior to produce a Twisted Failure + try: + # Simulate a Twisted inline callback returning a Response + # The returnValue method raises an exception encapsulating the value + returnValue(response) + except BaseException as exc: + def_gen_return_exc = exc + try: + # Simulate the media_downloaded callback raising a FileException + # This usually happens when the status code is not 200 OK + raise FileException('download-error') + except Exception as exc: + file_exc = exc + # Simulate Twisted capturing the FileException + # It encapsulates the exception inside a Twisted Failure + failure = Failure(file_exc) + + # The Failure should encapsulate a FileException ... + self.assertEqual(failure.value, file_exc) + # ... and if we're running on Python 3 ... + if sys.version_info.major >= 3: + # ... it should have the returnValue exception set as its context + self.assertEqual(failure.value.__context__, def_gen_return_exc) + + # Let's calculate the request fingerprint and fake some runtime data... + fp = request_fingerprint(request) + info = self.pipe.spiderinfo + info.downloading.add(fp) + info.waiting[fp] = [] + + # When calling the method that caches the Request's result ... + self.pipe._cache_result_and_execute_waiters(failure, fp, info) + # ... it should store the Twisted Failure ... + self.assertEqual(info.downloaded[fp], failure) + # ... encapsulating the original FileException ... + self.assertEqual(info.downloaded[fp].value, file_exc) + # ... but it should not store the returnValue exception on its context + context = getattr(info.downloaded[fp].value, '__context__', None) + self.assertIsNone(context) + class MockedMediaPipeline(MediaPipeline): From 8a3b15eb91169ab262e4dca60105f56467ecd1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 27 Mar 2019 08:50:33 +0100 Subject: [PATCH 0462/1853] Document how to select dynamically-loaded content --- docs/index.rst | 4 + docs/topics/dynamic-content.rst | 246 ++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 docs/topics/dynamic-content.rst diff --git a/docs/index.rst b/docs/index.rst index cedde8f38..6d5f9e77d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,6 +158,7 @@ Solving specific problems topics/practices topics/broad-crawls topics/developer-tools + topics/dynamic-content topics/leaks topics/media-pipeline topics/deploy @@ -183,6 +184,9 @@ Solving specific problems :doc:`topics/developer-tools` Learn how to scrape with your browser's developer tools. +:doc:`topics/dynamic-content` + Read webpage data that is loaded dynamically. + :doc:`topics/leaks` Learn how to find and get rid of memory leaks in your crawler. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst new file mode 100644 index 000000000..8b5dacf56 --- /dev/null +++ b/docs/topics/dynamic-content.rst @@ -0,0 +1,246 @@ +.. _topics-dynamic-content: + +==================================== +Selecting dynamically-loaded content +==================================== + +Some webpages show the desired data when you load them in a web browser. +However, when you download them using Scrapy, you cannot reach the desired data +using :ref:`selectors `. + +When this happens, the recommended approach is to +:ref:`find the data source ` and extract the data +from it. + +If you fail to do that, and you can nonetheless access the desired data through +the :ref:`DOM ` from your web browser, see +:ref:`topics-javascript-rendering`. + +.. _topics-finding-data-source: + +Finding the data source +======================= + +To extract the desired data, you must first find its source location. + +If the data is in a non-text-based format, such as an image or a PDF document, +use the :ref:`network tool ` of your web browser to find +the corresponding request, and :ref:`reproduce it +`. + +If your web browser lets you select the desired data as text, the data may be +defined in embedded JavaScript code, or loaded from an external resource in a +text-based format. + +In that case, you can use a tool like wgrep_ to find the URL of that resource. + +If the data turns out to come from the original URL itself, you must +:ref:`inspect the source code of the webpage ` to +determine where the data is located. + +If the data comes from a different URL, you will need to :ref:`reproduce the +corresponding request `. + +.. _topics-inspecting-source: + +Inspecting the source code of a webpage +======================================= + +Sometimes you need to inspect the source code of a webpage (not the +:ref:`DOM `) to determine where some desired data is located. + +Use Scrapy’s :command:`fetch` command to download the webpage contents as seen +by Scrapy:: + + scrapy fetch --nolog https://example.com > response.html + +If the desired data is in embedded JavaScript code within a `` {% endblock %} From d458ccff3b2d6df94df1aa86eeb7d2505d62f2d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 12:27:35 -0300 Subject: [PATCH 1680/1853] Retry request: priority_adjust cannot be float (Request.priority is int) --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 2721db7cf..5965a1c6c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -41,7 +41,7 @@ def get_retry_request( spider: Spider, reason: Union[str, Exception] = 'unspecified', max_retry_times: Optional[int] = None, - priority_adjust: Union[int, float, None] = None, + priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = 'retry', ): From 5492972d8a874e9fb7780175e4f396bc8f39378c Mon Sep 17 00:00:00 2001 From: anay2103 <55763427+anay2103@users.noreply.github.com> Date: Thu, 1 Apr 2021 20:30:48 +0300 Subject: [PATCH 1681/1853] added customized filtering examples in logging.rst (#4965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added customized filtering examples in logging.rst * Update logging.rst * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update logging.rst Co-authored-by: Adrián Chaves --- docs/topics/logging.rst | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index c3445d40e..00806392a 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method:: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. +You can also filter log records by :class:`~logging.LogRecord` data. For +example, you can filter log records by message content using a substring or +a regular expression. Create a :class:`logging.Filter` subclass +and equip it with a regular expression pattern to +filter out unwanted messages:: + + import logging + import re + + class ContentFilter(logging.Filter): + def filter(self, record): + match = re.search(r'\d{3} [Ee]rror, retrying', record.message) + if match: + return False + +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to +filter all loggers in different parts of the project +(middlewares, spider, etc.):: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + for handler in logging.root.handlers: + handler.addFilter(ContentFilter()) + +Alternatively, you may choose a specific logger +and hide it without affecting other loggers:: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('my_logger') + logger.addFilter(ContentFilter()) + scrapy.utils.log module ======================= From ad7456746961fb23beb3ebc051ee1ab0ed82a91e Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Thu, 1 Apr 2021 19:33:56 +0200 Subject: [PATCH 1682/1853] Fix argument type (int -> bool) (#4950) * Fix warning: Expected type 'bool', got 'int' instead * Update defer.py * Fix warning: Expected type 'bool', got 'int' instead Co-authored-by: Eugenio Lacuesta --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7f267c800..ef2eb3ba6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -71,7 +71,7 @@ def _urlencode(seq, enc): values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] - return urlencode(values, doseq=1) + return urlencode(values, doseq=True) def _get_form(response, formname, formid, formnumber, formxpath): From cc095aa8950975cf203b1d33724cc499043e3f17 Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Thu, 1 Apr 2021 23:09:33 +0530 Subject: [PATCH 1683/1853] Windows pip installation guide (#4736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added initial steps * fixing link * python3 -> python * remaining steps * steps updated * Update docs/intro/install.rst Co-authored-by: Adrián Chaves * added link to Visual Studio * removed 'install V' * Update docs/intro/install.rst Co-authored-by: Adrián Chaves Co-authored-by: Adrián Chaves --- docs/intro/install.rst | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index bf919ce25..8581dde0b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -12,6 +12,7 @@ Supported Python versions Scrapy requires Python 3.6+, either the CPython implementation (default) or the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +.. _intro-install-scrapy: Installing Scrapy ================= @@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -117,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++” for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -268,4 +290,6 @@ For details, see `Issue #2473 `_. .. _zsh: https://www.zsh.org/ .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ From 9e7cbc05ae10ab6c1335d92b4d41429ba8e2bf24 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 15:22:51 -0300 Subject: [PATCH 1684/1853] Fix type for urlencode's doseq argument --- scrapy/commands/bench.py | 2 +- tests/spiders.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 999c987ea..6bdf9eae0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' + url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/tests/spiders.py b/tests/spiders.py index 7e579098a..5b45f897e 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] def parse(self, response): @@ -245,7 +245,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 From 9e3b868dd83a3ac93dfe04b53d5130145154f478 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:04:09 +0500 Subject: [PATCH 1685/1853] Use __qualname__ in middleware handling. --- scrapy/core/downloader/middleware.py | 9 +++------ scrapy/core/spidermw.py | 10 +++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..177f3f760 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_request must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {response.__class__.__name__}" ) if response: @@ -55,8 +54,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_response must return Response or Request, " + f"Middleware {method.__qualname__} must return Response or Request, " f"got {type(response)}" ) if isinstance(response, Request): @@ -70,8 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_exception must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {type(response)}" ) if response: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 289292da7..e8733c4ad 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -18,10 +18,6 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -46,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -83,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -108,7 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {_fname(method)} must return an " + msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) From a9e96f99077865bc2d844ecf0ae174db97ba6d8b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:40:45 +0500 Subject: [PATCH 1686/1853] Add typing for middleware and coroutine related code. --- scrapy/core/downloader/middleware.py | 14 ++++++++----- scrapy/core/spidermw.py | 31 ++++++++++++++++++---------- scrapy/middleware.py | 11 +++++----- scrapy/utils/asyncgen.py | 5 ++++- scrapy/utils/defer.py | 23 ++++++++++++--------- scrapy/utils/python.py | 7 ++++--- 6 files changed, 56 insertions(+), 35 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..441fc9fa6 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,12 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from twisted.internet import defer +from typing import Callable, Union +from twisted.internet import defer +from twisted.python.failure import Failure + +from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager @@ -29,9 +33,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_exception'): self.methods['process_exception'].appendleft(mw.process_exception) - def download(self, download_func, request, spider): + def download(self, download_func: Callable, request: Request, spider: Spider): @defer.inlineCallbacks - def process_request(request): + def process_request(request: Request): for method in self.methods['process_request']: response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): @@ -45,7 +49,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return (yield download_func(request=request, spider=spider)) @defer.inlineCallbacks - def process_response(response): + def process_response(response: Union[Response, Request]): if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -64,7 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response @defer.inlineCallbacks - def process_exception(failure): + def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 289292da7..b09adf8e2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,25 +3,32 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from collections.abc import Iterable, AsyncIterable from itertools import islice +from typing import Callable, Union, Any from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput +from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') +def _isiterable(o): + return isinstance(o, Iterable) def _fname(f): return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" +ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -41,7 +48,7 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func, response, request, spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -55,7 +62,8 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, + exception_processor_index: int, recover_to: MutableChain): try: for r in iterable: yield r @@ -66,7 +74,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - def _process_spider_exception(self, response, spider, _failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -88,7 +96,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) return _failure - def _process_spider_output(self, response, spider, result, start_index=0): + def _process_spider_output(self, response: Response, spider: Spider, + result: Iterable, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -114,21 +123,21 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) - def _process_callback_output(self, response, spider, result): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable): recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) return MutableChain(self._process_spider_output(response, spider, result), recovered) - def scrape_response(self, scrape_func, response, request, spider): - def process_callback_output(result): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): + def process_callback_output(result: Iterable): return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure): + def process_spider_exception(_failure: Failure): return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider): + def process_start_requests(self, start_requests, spider: Spider): return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 5040378ea..c53cfb814 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,6 +1,7 @@ -from collections import defaultdict, deque import logging import pprint +from collections import defaultdict, deque +from typing import Callable from scrapy.exceptions import NotConfigured from scrapy.utils.misc import create_instance, load_object @@ -16,7 +17,7 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(deque) + self.methods: dict[str, deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -58,13 +59,13 @@ class MiddlewareManager: if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname, obj, *args): + def _process_parallel(self, methodname: str, obj, *args): return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname, obj, *args): + def _process_chain(self, methodname: str, obj, *args): return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args): return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 7f697af5f..c290e376c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,4 +1,7 @@ -async def collect_asyncgen(result): +from collections.abc import AsyncIterable + + +async def collect_asyncgen(result: AsyncIterable): results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6db9cc117..c382a00f7 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,16 +3,19 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect +from collections.abc import Coroutine from functools import wraps +from typing import Callable, Iterable, Any from twisted.internet import defer, task from twisted.python import failure +from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure): +def defer_fail(_failure: Failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -47,7 +50,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw): """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -64,7 +67,7 @@ def mustbe_deferred(f, *args, **kw): return defer_result(result) -def parallel(iterable, count, callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -75,7 +78,7 @@ def parallel(iterable, count, callable, *args, **named): return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks, input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() for x in callbacks: @@ -84,7 +87,7 @@ def process_chain(callbacks, input, *a, **kw): return d -def process_chain_both(callbacks, errbacks, input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): @@ -100,7 +103,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): return d -def process_parallel(callbacks, input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -110,7 +113,7 @@ def process_parallel(callbacks, input, *a, **kw): return d -def iter_errback(iterable, errback, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw): """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -124,7 +127,7 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o): +def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o @@ -139,7 +142,7 @@ def deferred_from_coro(o): return o -def deferred_f_from_coro_f(coro_f): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -151,7 +154,7 @@ def deferred_f_from_coro_f(coro_f): return f -def maybeDeferred_coro(f, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw): """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..bcc12f24f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,6 +8,7 @@ import re import sys import warnings import weakref +from collections.abc import Iterable from functools import partial, wraps from itertools import chain @@ -335,15 +336,15 @@ else: gc.collect() -class MutableChain: +class MutableChain(Iterable): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args): + def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables): + def extend(self, *iterables: Iterable): self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): From 414dd1119a7e15c612de7bd0fe560b6ca30b505f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:54:55 +0500 Subject: [PATCH 1687/1853] Drop an unused import. --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b09adf8e2..9a5305376 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from collections.abc import Iterable, AsyncIterable +from collections.abc import Iterable from itertools import islice from typing import Callable, Union, Any From 7dc857668f16e8c52ff44662aceb32f93ae3d80e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 4 Apr 2021 16:15:33 +0500 Subject: [PATCH 1688/1853] Also some typing for Scraper. --- scrapy/core/scraper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..4a3eff888 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,12 +3,14 @@ extracts information from them""" import logging from collections import deque +from collections.abc import Iterable +from typing import Union from itemadapter import is_item from twisted.internet import defer from twisted.python.failure import Failure -from scrapy import signals +from scrapy import signals, Spider from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -120,7 +122,7 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result, request, spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +133,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result, request, spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,7 +143,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result, request, spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -156,7 +158,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure, request, response, spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +179,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result, request, response, spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) From e7d51886ef90f5b2d4fa13911381680ac192fc37 Mon Sep 17 00:00:00 2001 From: Mayank Singhal <17mayanksinghal@gmail.com> Date: Tue, 6 Apr 2021 02:21:18 +0530 Subject: [PATCH 1689/1853] Find bash from PATH instead of /bin/bash --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index ff68bf1ae..87d5d3047 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ PYTHON = python SPHINXOPTS = PAPER = SOURCES = -SHELL = /bin/bash +SHELL = /usr/bin/env bash ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ -D latex_elements.papersize=$(PAPER) \ From a71d6ef29da8e3dfa906fbaa74ee1db64567f856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 6 Apr 2021 16:09:07 +0200 Subject: [PATCH 1690/1853] 2.5.0 release notes (#5028) Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- docs/news.rst | 193 ++++++++++++++++++++++++++++++- docs/topics/request-response.rst | 4 +- docs/topics/settings.rst | 5 +- docs/topics/signals.rst | 2 +- tests/test_webclient.py | 2 +- 5 files changed, 200 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index d9fe897ad..0ea412e75 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,190 @@ Release notes ============= +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + .. _release-2.4.1: Scrapy 2.4.1 (2020-11-17) @@ -97,6 +281,8 @@ Backward-incompatible changes (:issue:`4717`, :issue:`4823`) +.. _2.4-deprecation-removals: + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -1433,6 +1619,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1789,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -4184,7 +4374,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -4331,6 +4521,7 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c0283df01..500781c05 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -703,7 +703,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The ``protocol`` parameter. .. attribute:: Response.url @@ -809,7 +809,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The protocol that was used to download the response. For instance: "HTTP/1.0", "HTTP/1.1" diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9dcee64eb..f5dca824f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -677,6 +677,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +.. _http2: + The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update :setting:`DOWNLOAD_HANDLERS` as follows:: @@ -703,7 +705,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. - - No support for the :signal:`bytes_received` signal. + - No support for the :signal:`bytes_received` and + :signal:`headers_received` signals. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 98cfa606c..3d838fb63 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -403,7 +403,7 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.5 .. signal:: headers_received .. function:: headers_received(headers, request, spider) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index f935a8689..6e4cb9b6e 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -418,7 +418,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayloadDisabledCipher(self): if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): - self.skipTest("This does work in PyPy with cryptography<=2.3.1") + self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From e63188cbf753d560e43d8489c821bd6eb9fe54e9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 6 Apr 2021 19:13:32 +0500 Subject: [PATCH 1691/1853] =?UTF-8?q?Bump=20version:=202.4.1=20=E2=86=92?= =?UTF-8?q?=202.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 956c512cb..d9e4a2831 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.1 +current_version = 2.5.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 005119baa..437459cd9 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.1 +2.5.0 From 8603f9d7a5524b7709b4e8a8fc04a75a9a4f0ffe Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Tue, 6 Apr 2021 20:23:07 +0100 Subject: [PATCH 1692/1853] Apply changes to other examples in the same section. --- docs/topics/practices.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db1ed362e..15ac520e2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,8 +118,8 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy - from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -143,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -153,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() @@ -168,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -178,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) @defer.inlineCallbacks def crawl(): From 5a75b14a5fbbbd37c14aa7317761655ac7706b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 Apr 2021 12:33:37 +0200 Subject: [PATCH 1693/1853] docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17 --- .readthedocs.yml | 1 - docs/pip.txt | 3 --- docs/requirements.txt | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 docs/pip.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 2d781ae81..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,5 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/pip.txt b/docs/pip.txt deleted file mode 100644 index 095e53a0d..000000000 --- a/docs/pip.txt +++ /dev/null @@ -1,3 +0,0 @@ -# In pip 20.3-21.0, the default dependency resolver causes the build in -# ReadTheDocs to fail due to memory exhaustion or timeout. -pip<20.3 diff --git a/docs/requirements.txt b/docs/requirements.txt index 3d34b47da..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ Sphinx>=3.0 sphinx-hoverxref>=0.2b1 sphinx-notfound-page>=0.4 -sphinx_rtd_theme>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file From 91f81445524630843561c8a3c71b7fa0f081d783 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 00:30:18 -0300 Subject: [PATCH 1694/1853] Remove deprecated Spider.make_requests_from_url method --- scrapy/spiders/__init__.py | 27 ++------------------------- tests/test_spider.py | 33 --------------------------------- 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..d8248c606 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -4,14 +4,12 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ import logging -import warnings from typing import Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.utils.deprecate import method_is_overridden class Spider(object_ref): @@ -57,34 +55,13 @@ class Spider(object_ref): crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): - cls = self.__class__ if not self.start_urls and hasattr(self, 'start_url'): raise AttributeError( "Crawling could not start: 'start_urls' not found " "or empty (but found 'start_url' attribute instead, " "did you miss an 's'?)") - if method_is_overridden(cls, Spider, 'make_requests_from_url'): - warnings.warn( - "Spider.make_requests_from_url method is deprecated; it " - "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead " - f"(see {cls.__module__}.{cls.__name__}).", - ) - for url in self.start_urls: - yield self.make_requests_from_url(url) - else: - for url in self.start_urls: - yield Request(url, dont_filter=True) - - def make_requests_from_url(self, url): - """ This method is deprecated. """ - warnings.warn( - "Spider.make_requests_from_url method is deprecated: " - "it will be removed and not be called by the default " - "Spider.start_requests method in future Scrapy releases. " - "Please override Spider.start_requests method instead." - ) - return Request(url, dont_filter=True) + for url in self.start_urls: + yield Request(url, dont_filter=True) def _parse(self, response, **kwargs): return self.parse(response, **kwargs) diff --git a/tests/test_spider.py b/tests/test_spider.py index d23543f6a..a7c3ee048 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name='foo'), Spider) - def test_make_requests_from_url_deprecated(self): - class MySpider4(Spider): - name = 'spider1' - start_urls = ['http://example.com'] - - class MySpider5(Spider): - name = 'spider2' - start_urls = ['http://example.com'] - - def make_requests_from_url(self, url): - return Request(url + "/foo", dont_filter=True) - - with warnings.catch_warnings(record=True) as w: - # spider without overridden make_requests_from_url method - # doesn't issue a warning - spider1 = MySpider4() - self.assertEqual(len(list(spider1.start_requests())), 1) - self.assertEqual(len(w), 0) - - # spider without overridden make_requests_from_url method - # should issue a warning when called directly - request = spider1.make_requests_from_url("http://www.example.com") - self.assertTrue(isinstance(request, Request)) - self.assertEqual(len(w), 1) - - # spider with overridden make_requests_from_url issues a warning, - # but the method still works - spider2 = MySpider5() - requests = list(spider2.start_requests()) - self.assertEqual(len(requests), 1) - self.assertEqual(requests[0].url, 'http://example.com/foo') - self.assertEqual(len(w), 2) - class NoParseMethodSpiderTest(unittest.TestCase): From b6f77806b0ec414a28cfcbac3fa2d928040548d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Apr 2021 12:19:30 -0300 Subject: [PATCH 1695/1853] Engine tests: fix item class spider, add minimal type hints --- setup.cfg | 3 --- tests/test_engine.py | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/setup.cfg b/setup.cfg index 89b4ec57e..1fab6fe22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,9 +43,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_engine] -ignore_errors = True - [mypy-tests.test_exporters] ignore_errors = True diff --git a/tests/test_engine.py b/tests/test_engine.py index ef1204f94..c406d2577 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -56,7 +56,7 @@ class TestSpider(Spider): name_re = re.compile(r"

                (.*?)

                ", re.M) price_re = re.compile(r">Price: \$(.*?)<", re.M) - item_cls = TestItem + item_cls: type = TestItem def parse(self, response): xlink = LinkExtractor() @@ -66,15 +66,15 @@ class TestSpider(Spider): yield Request(url=link.url, callback=self.parse_item) def parse_item(self, response): - item = self.item_cls() + adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - item['name'] = m.group(1) - item['url'] = response.url + adapter['name'] = m.group(1) + adapter['url'] = response.url m = self.price_re.search(response.text) if m: - item['price'] = m.group(1) - return item + adapter['price'] = m.group(1) + return adapter.item class TestDupeFilterSpider(TestSpider): @@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider): class AttrsItemsSpider(TestSpider): - item_class = AttrsItem + item_cls = AttrsItem try: @@ -97,14 +97,12 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class DataClassItemsSpider(DictItemsSpider): + class _dataclass_spider(DictItemsSpider): def parse_item(self, response): item = super().parse_item(response) - return TestDataClass( - name=item.get('name'), - url=item.get('url'), - price=item.get('price'), - ) + return TestDataClass(**item) + + DataClassItemsSpider = _dataclass_spider class ItemZeroDivisionErrorSpider(TestSpider): From 4673f05ddec85f18a890e606a6b85282035e26be Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 9 Apr 2021 23:42:24 +0500 Subject: [PATCH 1696/1853] Cleanup of slot handling in Scraper. --- scrapy/core/scraper.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..4a3ddea10 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -82,28 +82,26 @@ class Scraper: def close_spider(self, spider): """Close a spider being scraped and release its resources""" - slot = self.slot - slot.closing = defer.Deferred() - slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider, slot) - return slot.closing + self.slot.closing = defer.Deferred() + self.slot.closing.addCallback(self.itemproc.close_spider) + self._check_if_closing(spider) + return self.slot.closing def is_idle(self): """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider, slot): - if slot.closing and slot.is_idle(): - slot.closing.callback(spider) + def _check_if_closing(self, spider): + if self.slot.closing and self.slot.is_idle(): + self.slot.closing.callback(spider) def enqueue_scrape(self, response, request, spider): - slot = self.slot - dfd = slot.add_response_request(response, request) + dfd = self.slot.add_response_request(response, request) def finish_scraping(_): - slot.finish_response(response, request) - self._check_if_closing(spider, slot) - self._scrape_next(spider, slot) + self.slot.finish_response(response, request) + self._check_if_closing(spider) + self._scrape_next(spider) return _ dfd.addBoth(finish_scraping) @@ -112,12 +110,12 @@ class Scraper: {'request': request}, exc_info=failure_to_exc_info(f), extra={'spider': spider})) - self._scrape_next(spider, slot) + self._scrape_next(spider) return dfd - def _scrape_next(self, spider, slot): - while slot.queue: - response, request, deferred = slot.next_response_request_deferred() + def _scrape_next(self, spider): + while self.slot.queue: + response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) def _scrape(self, result, request, spider): From d8d1dc5b508832598591ae9d3d3694f77f70492d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 12 Apr 2021 10:43:02 -0300 Subject: [PATCH 1697/1853] Ignore typing warning in test --- tests/test_engine.py | 4 +--- tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c406d2577..b2d1d83c7 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -97,13 +97,11 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class _dataclass_spider(DictItemsSpider): + class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] def parse_item(self, response): item = super().parse_item(response) return TestDataClass(**item) - DataClassItemsSpider = _dataclass_spider - class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { diff --git a/tox.ini b/tox.ini index 353977519..5b0606f8f 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ basepython = python3 deps = mypy==0.780 commands = - mypy {posargs: scrapy tests} + mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3 From a4415e4e6fa0f9becb40968438a77f4ea262633a Mon Sep 17 00:00:00 2001 From: Mayank Singhal <17mayanksinghal@gmail.com> Date: Tue, 13 Apr 2021 17:20:55 +0530 Subject: [PATCH 1698/1853] Add DataURI download handler in DOWNLOAD_HANDLERS_BASE documentation --- docs/topics/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f5dca824f..1d5babcec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -657,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE Default:: { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', From 76fa2257ef0280fc82e123457c791254cc2f185e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 20:01:18 +0500 Subject: [PATCH 1699/1853] Add typing also for return values, other small fixes. --- scrapy/core/spidermw.py | 33 ++++++++++++++++++--------------- scrapy/middleware.py | 24 ++++++++++++++---------- scrapy/utils/defer.py | 24 ++++++++++++------------ 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dc0e58095..05df8c988 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,10 +3,10 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from collections.abc import Iterable from itertools import islice -from typing import Callable, Union, Any +from typing import Callable, Union, Any, Generator, Iterable +from twisted.internet.defer import Deferred from twisted.python.failure import Failure from scrapy import Request, Spider @@ -18,13 +18,13 @@ from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(o): - return isinstance(o, Iterable) - - ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] +def _isiterable(o) -> bool: + return isinstance(o, Iterable) + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -44,7 +44,8 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Any: for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -59,7 +60,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, - exception_processor_index: int, recover_to: MutableChain): + exception_processor_index: int, recover_to: MutableChain) -> Generator: try: for r in iterable: yield r @@ -70,7 +71,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, + start_index: int = 0) -> Union[Failure, MutableChain]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -93,7 +95,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def _process_spider_output(self, response: Response, spider: Spider, - result: Iterable, start_index=0): + result: Iterable, start_index: int = 0) -> MutableChain: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -119,21 +121,22 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) - def _process_callback_output(self, response: Response, spider: Spider, result: Iterable): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) return MutableChain(self._process_spider_output(response, spider, result), recovered) - def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): - def process_callback_output(result: Iterable): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Deferred: + def process_callback_output(result: Iterable) -> MutableChain: return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure: Failure): + def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider: Spider): + def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index c53cfb814..3f8c1cbf5 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,9 +1,13 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable +from typing import Callable, Dict, Deque +from twisted.internet import defer + +from scrapy import Spider from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object from scrapy.utils.defer import process_parallel, process_chain, process_chain_both @@ -17,16 +21,16 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: dict[str, deque[Callable]] = defaultdict(deque) + self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> list: raise NotImplementedError @classmethod - def from_settings(cls, settings, crawler=None): + def from_settings(cls, settings: Settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -53,24 +57,24 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw): + def _add_middleware(self, mw) -> None: if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args): + def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname: str, obj, *args): + def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> defer.Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> defer.Deferred: return self._process_parallel('close_spider', spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c382a00f7..095eae94c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,7 +5,7 @@ import asyncio import inspect from collections.abc import Coroutine from functools import wraps -from typing import Callable, Iterable, Any +from typing import Callable, Iterable, Any, Generator from twisted.internet import defer, task from twisted.python import failure @@ -15,7 +15,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure: Failure): +def defer_fail(_failure: Failure) -> defer.Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -28,7 +28,7 @@ def defer_fail(_failure: Failure): return d -def defer_succeed(result): +def defer_succeed(result) -> defer.Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -41,7 +41,7 @@ def defer_succeed(result): return d -def defer_result(result): +def defer_result(result) -> defer.Deferred: if isinstance(result, defer.Deferred): return result elif isinstance(result, failure.Failure): @@ -50,7 +50,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -67,7 +67,7 @@ def mustbe_deferred(f: Callable, *args, **kw): return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -78,7 +78,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks: Iterable[Callable], input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() for x in callbacks: @@ -87,7 +87,7 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw): return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): @@ -103,7 +103,7 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -113,7 +113,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -142,7 +142,7 @@ def deferred_from_coro(o) -> Any: return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -154,7 +154,7 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): return f -def maybeDeferred_coro(f: Callable, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) From 335a25675278543d4c85123bbbed99f228b26416 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:20 +0500 Subject: [PATCH 1700/1853] Update scrapy/core/spidermw.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 05df8c988..7e58521ac 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Callable, Union, Any, Generator, Iterable +from typing import Any, Callable, Generator, Iterable, Union from twisted.internet.defer import Deferred from twisted.python.failure import Failure From b0e75125749ec1dce14614468bf06cd8e8842649 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:25 +0500 Subject: [PATCH 1701/1853] Update scrapy/middleware.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 3f8c1cbf5..09768b59d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Dict, Deque +from typing import Callable, Deque, Dict from twisted.internet import defer From a8de04c823f5e10d012aaa2dd94a4f5a9f70b119 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:30 +0500 Subject: [PATCH 1702/1853] Update scrapy/utils/defer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 095eae94c..e1139b1d1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,7 +5,7 @@ import asyncio import inspect from collections.abc import Coroutine from functools import wraps -from typing import Callable, Iterable, Any, Generator +from typing import Any, Callable, Generator, Iterable from twisted.internet import defer, task from twisted.python import failure From cef0a8b3d653d847efe32dfc2850e5992b627408 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 21:07:07 +0500 Subject: [PATCH 1703/1853] Import Deferred directly in scrapy/middleware.py. --- scrapy/middleware.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 09768b59d..bbec38086 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,7 +3,7 @@ import pprint from collections import defaultdict, deque from typing import Callable, Deque, Dict -from twisted.internet import defer +from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.exceptions import NotConfigured @@ -63,18 +63,18 @@ class MiddlewareManager: if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred: + def _process_parallel(self, methodname: str, obj, *args) -> Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred: + def _process_chain(self, methodname: str, obj, *args) -> Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred: + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider: Spider) -> defer.Deferred: + def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider: Spider) -> defer.Deferred: + def close_spider(self, spider: Spider) -> Deferred: return self._process_parallel('close_spider', spider) From 08e4eaf97369ba6daa4b5d84e00fd4d36b78e00a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 22:41:01 +0500 Subject: [PATCH 1704/1853] Import Deferred directly in scrapy/utils/defer.py. --- scrapy/utils/defer.py | 48 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index e1139b1d1..b317c12a3 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -7,7 +7,9 @@ from collections.abc import Coroutine from functools import wraps from typing import Any, Callable, Generator, Iterable -from twisted.internet import defer, task +from twisted.internet import defer +from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.task import Cooperator from twisted.python import failure from twisted.python.failure import Failure @@ -15,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure: Failure) -> defer.Deferred: +def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -23,12 +25,12 @@ def defer_fail(_failure: Failure) -> defer.Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.errback, _failure) return d -def defer_succeed(result) -> defer.Deferred: +def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -36,13 +38,13 @@ def defer_succeed(result) -> defer.Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result) -> defer.Deferred: - if isinstance(result, defer.Deferred): +def defer_result(result) -> Deferred: + if isinstance(result, Deferred): return result elif isinstance(result, failure.Failure): return defer_fail(result) @@ -50,7 +52,7 @@ def defer_result(result) -> defer.Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: +def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -67,29 +69,29 @@ def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList: +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ - coop = task.Cooperator() + coop = Cooperator() work = (callable(elem, *args, **named) for elem in iterable) - return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = defer.Deferred() + d = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = defer.Deferred() + d = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, errback=eb, @@ -103,12 +105,12 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d @@ -129,16 +131,16 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" - if isinstance(o, defer.Deferred): + if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return defer.ensureDeferred(o) + return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return defer.Deferred.fromFuture(asyncio.ensure_future(o)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o @@ -154,14 +156,14 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred: +def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) except: # noqa: E722 - return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + return defer.fail(failure.Failure(captureVars=Deferred.debug)) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): return result elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) From 77bff0db0a6bdfca15295444f0e0eda47b35e702 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 19:10:13 +0500 Subject: [PATCH 1705/1853] Additional typing for scraper and a small code change. --- scrapy/core/scraper.py | 100 ++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c760a4155..adbf8ef3d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,11 +3,10 @@ extracts information from them""" import logging from collections import deque -from collections.abc import Iterable -from typing import Union +from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable from itemadapter import is_item -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import signals, Spider @@ -20,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred] + + logger = logging.getLogger(__name__) @@ -28,46 +30,46 @@ class Slot: MIN_RESPONSE_SIZE = 1024 - def __init__(self, max_active_size=5000000): + def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue = deque() - self.active = set() - self.active_size = 0 - self.itemproc_size = 0 - self.closing = None + self.queue: Deque[QUEUE_TUPLE] = deque() + self.active: Set[Request] = set() + self.active_size: int = 0 + self.itemproc_size: int = 0 + self.closing: Optional[Deferred] = None - def add_response_request(self, response, request): - deferred = defer.Deferred() - self.queue.append((response, request, deferred)) - if isinstance(response, Response): - self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE) + def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred: + deferred = Deferred() + self.queue.append((result, request, deferred)) + if isinstance(result, Response): + self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self): + def next_response_request_deferred(self) -> QUEUE_TUPLE: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred - def finish_response(self, response, request): + def finish_response(self, result: Union[Response, Failure], request: Request) -> None: self.active.remove(request) - if isinstance(response, Response): - self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE) + if isinstance(result, Response): + self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size -= self.MIN_RESPONSE_SIZE - def is_idle(self): + def is_idle(self) -> bool: return not (self.queue or self.active) - def needs_backout(self): + def needs_backout(self) -> bool: return self.active_size > self.max_active_size class Scraper: def __init__(self, crawler): - self.slot = None + self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) self.itemproc = itemproc_cls.from_crawler(crawler) @@ -76,32 +78,37 @@ class Scraper: self.signals = crawler.signals self.logformatter = crawler.logformatter - @defer.inlineCallbacks - def open_spider(self, spider): + @inlineCallbacks + def open_spider(self, spider: Spider): """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE')) yield self.itemproc.open_spider(spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> Deferred: """Close a spider being scraped and release its resources""" - self.slot.closing = defer.Deferred() + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + self.slot.closing = Deferred() self.slot.closing.addCallback(self.itemproc.close_spider) self._check_if_closing(spider) return self.slot.closing - def is_idle(self): + def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider): + def _check_if_closing(self, spider: Spider) -> None: + assert self.slot is not None # typing if self.slot.closing and self.slot.is_idle(): self.slot.closing.callback(spider) - def enqueue_scrape(self, response, request, spider): - dfd = self.slot.add_response_request(response, request) + def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + dfd = self.slot.add_response_request(result, request) def finish_scraping(_): - self.slot.finish_response(response, request) + self.slot.finish_response(result, request) self._check_if_closing(spider) self._scrape_next(spider) return _ @@ -115,12 +122,13 @@ class Scraper: self._scrape_next(spider) return dfd - def _scrape_next(self, spider): + def _scrape_next(self, spider: Spider) -> None: + assert self.slot is not None # typing while self.slot.queue: response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +139,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,7 +149,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -156,7 +164,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +185,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) @@ -185,10 +193,12 @@ class Scraper: request, response, spider) return dfd - def _process_spidermw_output(self, output, request, response, spider): + def _process_spidermw_output(self, output: Any, request: Request, response: Response, + spider: Spider) -> Optional[Deferred]: """Process each Request/Item (given in the output parameter) returned from the given spider """ + assert self.slot is not None # typing if isinstance(output, Request): self.crawler.engine.crawl(request=output, spider=spider) elif is_item(output): @@ -205,12 +215,18 @@ class Scraper: {'request': request, 'typename': typename}, extra={'spider': spider}, ) + return None - def _log_download_errors(self, spider_failure, download_failure, request, spider): + def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request, + spider: Spider) -> Union[Failure, None]: """Log and silence errors that come from the engine (typically download - errors that got propagated thru here) + errors that got propagated thru here). + + spider_failure: the value passed into the errback of self.call_spider() + download_failure: the value passed into _scrape2() from + ExecutionEngine._handle_downloader_output() as "result" """ - if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): + if not download_failure.check(IgnoreRequest): if download_failure.frames: logkws = self.logformatter.download_error(download_failure, request, spider) logger.log( @@ -230,10 +246,12 @@ class Scraper: if spider_failure is not download_failure: return spider_failure + return None - def _itemproc_finished(self, output, item, response, spider): + def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None: """ItemProcessor finished for the given ``item`` and returned ``output`` """ + assert self.slot is not None # typing self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value From 309a637f32b0f6196eb4f77f19152527dcf3e5e7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 20:26:37 +0500 Subject: [PATCH 1706/1853] Small changes. --- scrapy/core/scraper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index adbf8ef3d..96aa53686 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,7 +3,7 @@ extracts information from them""" import logging from collections import deque -from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable +from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -19,7 +19,7 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output -QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred] +QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class Slot: def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue: Deque[QUEUE_TUPLE] = deque() + self.queue: Deque[QueueTuple] = deque() self.active: Set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 @@ -47,7 +47,7 @@ class Slot: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self) -> QUEUE_TUPLE: + def next_response_request_deferred(self) -> QueueTuple: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred From 7e23677b52b659b11471a63f3be9905a0bbaf995 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 08:45:28 -0300 Subject: [PATCH 1707/1853] Engine: deprecations and type hints (#5090) --- docs/topics/telnetconsole.rst | 3 +- scrapy/core/engine.py | 419 ++++++++++--------- scrapy/core/scraper.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/extensions/memusage.py | 6 +- scrapy/pipelines/media.py | 2 +- scrapy/shell.py | 2 +- scrapy/utils/engine.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 12 +- tests/test_engine.py | 110 ++++- 10 files changed, 336 insertions(+), 225 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 9802a34a2..832829b75 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -110,11 +110,10 @@ using the telnet console:: Execution engine status time()-engine.start_time : 8.62972998619 - engine.has_capacity() : False len(engine.downloader.active) : 16 engine.scraper.is_idle() : False engine.spider.name : followall - engine.spider_is_idle(engine.spider) : False + engine.spider_is_idle() : False engine.slot.closing : False len(engine.slot.inprogress) : 16 len(engine.slot.scheduler.dqs or []) : 0 diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 93bcdb49a..edfac87c6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -1,51 +1,61 @@ """ -This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. +This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ import logging +import warnings from time import time +from typing import Callable, Iterable, Iterator, Optional, Set, Union -from twisted.internet import defer, task +from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request -from scrapy.utils.misc import load_object -from scrapy.utils.reactor import CallLaterOnce +from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.reactor import CallLaterOnce + logger = logging.getLogger(__name__) class Slot: - - def __init__(self, start_requests, close_if_idle, nextcall, scheduler): - self.closing = False - self.inprogress = set() # requests in progress - self.start_requests = iter(start_requests) + def __init__( + self, + start_requests: Iterable, + close_if_idle: bool, + nextcall: CallLaterOnce, + scheduler, + ) -> None: + self.closing: Optional[Deferred] = None + self.inprogress: Set[Request] = set() + self.start_requests: Optional[Iterator] = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall self.scheduler = scheduler - self.heartbeat = task.LoopingCall(nextcall.schedule) + self.heartbeat = LoopingCall(nextcall.schedule) - def add_request(self, request): + def add_request(self, request: Request) -> None: self.inprogress.add(request) - def remove_request(self, request): + def remove_request(self, request: Request) -> None: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self): - self.closing = defer.Deferred() + def close(self) -> Deferred: + self.closing = Deferred() self._maybe_fire_closing() return self.closing - def _maybe_fire_closing(self): - if self.closing and not self.inprogress: + def _maybe_fire_closing(self) -> None: + if self.closing is not None and not self.inprogress: if self.nextcall: self.nextcall.cancel() if self.heartbeat.running: @@ -54,210 +64,224 @@ class Slot: class ExecutionEngine: - - def __init__(self, crawler, spider_closed_callback): + def __init__(self, crawler, spider_closed_callback: Callable) -> None: self.crawler = crawler self.settings = crawler.settings self.signals = crawler.signals self.logformatter = crawler.logformatter - self.slot = None - self.spider = None + self.slot: Optional[Slot] = None + self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(self.settings['SCHEDULER']) + self.scheduler_cls = load_object(crawler.settings["SCHEDULER"]) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback - @defer.inlineCallbacks - def start(self): - """Start the execution engine""" + @inlineCallbacks + def start(self) -> Deferred: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = defer.Deferred() + self._closewait = Deferred() yield self._closewait - def stop(self): - """Stop the execution engine gracefully""" + def stop(self) -> Deferred: + """Gracefully stop the execution engine""" + @inlineCallbacks + def _finish_stopping_engine(_) -> Deferred: + yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + self._closewait.callback(None) + if not self.running: raise RuntimeError("Engine not running") + self.running = False - dfd = self._close_all_spiders() - return dfd.addBoth(lambda _: self._finish_stopping_engine()) + dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None) + return dfd.addBoth(_finish_stopping_engine) - def close(self): - """Close the execution engine gracefully. - - If it has already been started, stop it. In all cases, close all spiders - and the downloader. + def close(self) -> Deferred: + """ + Gracefully close the execution engine. + If it has already been started, stop it. In all cases, close the spider and the downloader. """ if self.running: - # Will also close spiders and downloader - return self.stop() - elif self.open_spiders: - # Will also close downloader - return self._close_all_spiders() - else: - return defer.succeed(self.downloader.close()) + return self.stop() # will also close spider and downloader + if self.spider is not None: + return self.close_spider(self.spider, reason="shutdown") # will also close downloader + return succeed(self.downloader.close()) - def pause(self): - """Pause the execution engine""" + def pause(self) -> None: self.paused = True - def unpause(self): - """Resume the execution engine""" + def unpause(self) -> None: self.paused = False - def _next_request(self, spider): - slot = self.slot - if not slot: - return + def _next_request(self) -> None: + assert self.slot is not None # typing + assert self.spider is not None # typing if self.paused: - return + return None - while not self._needs_backout(spider): - if not self._next_request_from_scheduler(spider): - break + while not self._needs_backout() and self._next_request_from_scheduler() is not None: + pass - if slot.start_requests and not self._needs_backout(spider): + if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(slot.start_requests) + request = next(self.slot.start_requests) except StopIteration: - slot.start_requests = None + self.slot.start_requests = None except Exception: - slot.start_requests = None - logger.error('Error while obtaining start requests', - exc_info=True, extra={'spider': spider}) + self.slot.start_requests = None + logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider}) else: - self.crawl(request, spider) + self.crawl(request) - if self.spider_is_idle(spider) and slot.close_if_idle: - self._spider_idle(spider) + if self.spider_is_idle() and self.slot.close_if_idle: + self._spider_idle() - def _needs_backout(self, spider): - slot = self.slot + def _needs_backout(self) -> bool: return ( not self.running - or slot.closing + or self.slot.closing # type: ignore[union-attr] or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() + or self.scraper.slot.needs_backout() # type: ignore[union-attr] ) - def _next_request_from_scheduler(self, spider): - slot = self.slot - request = slot.scheduler.next_request() - if not request: - return - d = self._download(request, spider) - d.addBoth(self._handle_downloader_output, request, spider) + def _next_request_from_scheduler(self) -> Optional[Deferred]: + assert self.slot is not None # typing + assert self.spider is not None # typing + + request = self.slot.scheduler.next_request() + if request is None: + return None + + d = self._download(request, self.spider) + d.addBoth(self._handle_downloader_output, request, self.spider) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.remove_request(request)) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.remove_request(request)) d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.nextcall.schedule()) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + extra={'spider': self.spider})) return d - def _handle_downloader_output(self, response, request, spider): - if not isinstance(response, (Request, Response, Failure)): - raise TypeError( - "Incorrect type: expected Request, Response or Failure, got " - f"{type(response)}: {response!r}" - ) + def _handle_downloader_output( + self, result: Union[Request, Response, Failure], request: Request, spider: Spider + ) -> Optional[Deferred]: + if not isinstance(result, (Request, Response, Failure)): + raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") + # downloader middleware can return requests (for example, redirects) - if isinstance(response, Request): - self.crawl(response, spider) - return - # response is a Response or Failure - d = self.scraper.enqueue_scrape(response, request, spider) - d.addErrback(lambda f: logger.error('Error while enqueuing downloader output', - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + if isinstance(result, Request): + self.crawl(result) + return None + + d = self.scraper.enqueue_scrape(result, request, spider) + d.addErrback( + lambda f: logger.error( + "Error while enqueuing downloader output", + exc_info=failure_to_exc_info(f), + extra={'spider': spider}, + ) + ) return d - def spider_is_idle(self, spider): - if not self.scraper.slot.is_idle(): - # scraper is not idle + def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False - - if self.downloader.active: - # downloader has pending requests + if self.downloader.active: # downloader has pending requests return False - - if self.slot.start_requests is not None: - # not all start requests are handled + if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - # scheduler has pending requests return False - return True - @property - def open_spiders(self): - return [self.spider] if self.spider else [] + def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + """Inject the request into the spider <-> downloader pipeline""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + raise RuntimeError(f"The spider {spider.name!r} does not match the open spider") + if self.spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + self._schedule_request(request, self.spider) + self.slot.nextcall.schedule() # type: ignore[union-attr] - def has_capacity(self): - """Does the engine have capacity to handle more spiders""" - return not bool(self.slot) - - def crawl(self, request, spider): - if spider not in self.open_spiders: - raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") - self.schedule(request, spider) - self.slot.nextcall.schedule() - - def schedule(self, request, spider): + def _schedule_request(self, request: Request, spider: Spider) -> None: self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) - if not self.slot.scheduler.enqueue_request(request): + if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) - def download(self, request, spider): - d = self._download(request, spider) - d.addBoth(self._downloaded, self.slot, request, spider) - return d + def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" + if spider is None: + spider = self.spider + else: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + logger.warning("The spider '%s' does not match the open spider", spider.name) + if spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + return self._download(request, spider).addBoth(self._downloaded, request, spider) - def _downloaded(self, response, slot, request, spider): - slot.remove_request(request) - return self.download(response, spider) if isinstance(response, Request) else response + def _downloaded( + self, result: Union[Response, Request], request: Request, spider: Spider + ) -> Union[Deferred, Response]: + assert self.slot is not None # typing + self.slot.remove_request(request) + return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request, spider): - slot = self.slot - slot.add_request(request) + def _download(self, request: Request, spider: Spider) -> Deferred: + assert self.slot is not None # typing - def _on_success(response): - if not isinstance(response, (Response, Request)): - raise TypeError( - "Incorrect type: expected Response or Request, got " - f"{type(response)}: {response!r}" - ) - if isinstance(response, Response): - if response.request is None: - response.request = request - logkws = self.logformatter.crawled(response.request, response, spider) + self.slot.add_request(request) + + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + if not isinstance(result, (Response, Request)): + raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") + if isinstance(result, Response): + if result.request is None: + result.request = request + logkws = self.logformatter.crawled(result.request, result, spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) self.signals.send_catch_log( signal=signals.response_received, - response=response, - request=response.request, + response=result, + request=result.request, spider=spider, ) - return response + return result def _on_complete(_): - slot.nextcall.schedule() + self.slot.nextcall.schedule() return _ dwld = self.downloader.fetch(request, spider) @@ -265,58 +289,52 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @defer.inlineCallbacks - def open_spider(self, spider, start_requests=(), close_if_idle=True): - if not self.has_capacity(): + @inlineCallbacks + def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True): + if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) - nextcall = CallLaterOnce(self._next_request, spider) - scheduler = self.scheduler_cls.from_crawler(self.crawler) + nextcall = CallLaterOnce(self._next_request) + scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) - slot = Slot(start_requests, close_if_idle, nextcall, scheduler) - self.slot = slot + self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - slot.nextcall.schedule() - slot.heartbeat.start(5) + self.slot.nextcall.schedule() + self.slot.heartbeat.start(5) - def _spider_idle(self, spider): - """Called when a spider gets idle. This function is called when there - are no remaining pages to download or schedule. It can be called - multiple times. If some extension raises a DontCloseSpider exception - (in the spider_idle signal handler) the spider is not closed until the - next loop and this function is guaranteed to be called (at least) once - again for this spider. + def _spider_idle(self) -> None: """ - res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) + Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. + It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider + exception, the spider is not closed until the next loop and this function is guaranteed to be called + (at least) once again. + """ + assert self.spider is not None # typing + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): - return + return None + if self.spider_is_idle(): + self.close_spider(self.spider, reason='finished') - if self.spider_is_idle(spider): - self.close_spider(spider, reason='finished') - - def close_spider(self, spider, reason='cancelled'): + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" + if self.slot is None: + raise RuntimeError("Engine slot not assigned") - slot = self.slot - if slot.closing: - return slot.closing - logger.info("Closing spider (%(reason)s)", - {'reason': reason}, - extra={'spider': spider}) + if self.slot.closing is not None: + return self.slot.closing - dfd = slot.close() + logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider}) - def log_failure(msg): - def errback(failure): - logger.error( - msg, - exc_info=failure_to_exc_info(failure), - extra={'spider': spider} - ) + dfd = self.slot.close() + + def log_failure(msg: str) -> Callable: + def errback(failure: Failure) -> None: + logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider}) return errback dfd.addBoth(lambda _: self.downloader.close()) @@ -325,19 +343,18 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: slot.scheduler.close(reason)) + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) dfd.addErrback(log_failure('Scheduler close failure')) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( - signal=signals.spider_closed, spider=spider, reason=reason)) + signal=signals.spider_closed, spider=spider, reason=reason, + )) dfd.addErrback(log_failure('Error while sending spider_close signal')) dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) dfd.addErrback(log_failure('Stats close failure')) - dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", - {'reason': reason}, - extra={'spider': spider})) + dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider})) dfd.addBoth(lambda _: setattr(self, 'slot', None)) dfd.addErrback(log_failure('Error while unassigning slot')) @@ -349,12 +366,26 @@ class ExecutionEngine: return dfd - def _close_all_spiders(self): - dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders] - dlist = defer.DeferredList(dfds) - return dlist + @property + def open_spiders(self) -> list: + warnings.warn( + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return [self.spider] if self.spider is not None else [] - @defer.inlineCallbacks - def _finish_stopping_engine(self): - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - self._closewait.callback(None) + def has_capacity(self) -> bool: + warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2) + return not bool(self.slot) + + def schedule(self, request: Request, spider: Spider) -> None: + warnings.warn( + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + self._schedule_request(request, spider) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 96aa53686..d6d6f64f9 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -200,7 +200,7 @@ class Scraper: """ assert self.slot is not None # typing if isinstance(output, Request): - self.crawler.engine.crawl(request=output, spider=spider) + self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index d6da55535..e66bf177e 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 274cbdbfe..9de119a10 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -88,10 +88,8 @@ class MemoryUsage: self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) - open_spiders = self.crawler.engine.open_spiders - if open_spiders: - for spider in open_spiders: - self.crawler.engine.close_spider(spider, 'memusage_exceeded') + if self.crawler.engine.spider is not None: + self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0c2ee6856..d1bccf323 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -173,7 +173,7 @@ class MediaPipeline: errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) - dfd = self.crawler.engine.download(request, info.spider) + dfd = self.crawler.engine.download(request) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) diff --git a/scrapy/shell.py b/scrapy/shell.py index c370ccaff..f2dff2ae3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -79,7 +79,7 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - self.crawler.engine.crawl(request, spider) + self.crawler.engine.crawl(request) return d def _open_spider(self, request, spider): diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0c1cee1a0..8e3ec2c37 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -8,11 +8,10 @@ def get_engine_status(engine): """Return a report of the current engine status""" tests = [ "time()-engine.start_time", - "engine.has_capacity()", "len(engine.downloader.active)", "engine.scraper.is_idle()", "engine.spider.name", - "engine.spider_is_idle(engine.spider)", + "engine.spider_is_idle()", "engine.slot.closing", "len(engine.slot.inprogress)", "len(engine.slot.scheduler.dqs or [])", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 858138f81..1460d88eb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -42,7 +42,7 @@ Disallow: /some/randome/page.html """.encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -79,7 +79,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,7 +102,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -122,7 +122,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def return_failure(request, spider): + def return_failure(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -138,7 +138,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def immediate_failure(request, spider): + def immediate_failure(request): deferred = Deferred() deferred.errback(failure.Failure(err)) return deferred @@ -150,7 +150,7 @@ Disallow: /some/randome/page.html def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) - def ignore_request(request, spider): + def ignore_request(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_engine.py b/tests/test_engine.py index b2d1d83c7..c200ded90 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,6 +13,7 @@ module with the ``runserver`` argument:: import os import re import sys +import warnings from collections import defaultdict from urllib.parse import urlparse @@ -25,6 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -382,22 +384,104 @@ class EngineTest(unittest.TestCase): yield e.close() @defer.inlineCallbacks - def test_close_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): + def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) + yield self.assertFailure(e.start(), RuntimeError).addBoth( + lambda exc: self.assertEqual(str(exc), "Engine already running") + ) + yield e.stop() + + @defer.inlineCallbacks + def test_close_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + self.assertEqual(len(e.open_spiders), 1) + yield e.close() + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_close_engine_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + e.start() + self.assertTrue(e.running) + yield e.close() + self.assertFalse(e.running) + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_crawl_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.crawl(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + ) + + @defer.inlineCallbacks + def test_download_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.download(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + ) + + @defer.inlineCallbacks + def test_deprecated_schedule(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.schedule(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + ) + + @defer.inlineCallbacks + def test_deprecated_has_capacity(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + self.assertTrue(e.has_capacity()) + spider = TestSpider() + yield e.open_spider(spider, []) + self.assertFalse(e.has_capacity()) + e.start() + yield e.close() + self.assertTrue(e.has_capacity()) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") if __name__ == "__main__": From e3f81d8d5f17515b6eba135ac0db7e270ff0a9f0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 11:46:43 -0300 Subject: [PATCH 1708/1853] Engine: remove unnecessary parameter (#5106) --- scrapy/core/engine.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index edfac87c6..7a09bafa1 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -161,7 +161,7 @@ class ExecutionEngine: return None d = self._download(request, self.spider) - d.addBoth(self._handle_downloader_output, request, self.spider) + d.addBoth(self._handle_downloader_output, request) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) @@ -176,8 +176,10 @@ class ExecutionEngine: return d def _handle_downloader_output( - self, result: Union[Request, Response, Failure], request: Request, spider: Spider + self, result: Union[Request, Response, Failure], request: Request ) -> Optional[Deferred]: + assert self.spider is not None # typing + if not isinstance(result, (Request, Response, Failure)): raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") @@ -186,12 +188,12 @@ class ExecutionEngine: self.crawl(result) return None - d = self.scraper.enqueue_scrape(result, request, spider) + d = self.scraper.enqueue_scrape(result, request, self.spider) d.addErrback( lambda f: logger.error( "Error while enqueuing downloader output", exc_info=failure_to_exc_info(f), - extra={'spider': spider}, + extra={'spider': self.spider}, ) ) return d From e779ed7d93beec36f565d33a1cc8d3e8fe6068d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 16:39:07 -0300 Subject: [PATCH 1709/1853] Dupefilter type hints (#5108) --- scrapy/dupefilters.py | 41 +++++++++++++++++++++++++++-------------- scrapy/utils/request.py | 2 +- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ac5478e7c..292c68099 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,35 +1,47 @@ -import os import logging +import os +from typing import Optional, Set, Type, TypeVar +from twisted.internet.defer import Deferred + +from scrapy.http.request import Request +from scrapy.settings import BaseSettings +from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint -class BaseDupeFilter: +BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") + +class BaseDupeFilter: @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV: return cls() - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: return False - def open(self): # can return deferred + def open(self) -> Optional[Deferred]: pass - def close(self, reason): # can return a deferred + def close(self, reason: str) -> Optional[Deferred]: pass - def log(self, request, spider): # log that a request has been filtered + def log(self, request: Request, spider: Spider) -> None: + """Log that a request has been filtered""" pass +RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") + + class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" - def __init__(self, path=None, debug=False): + def __init__(self, path: Optional[str] = None, debug: bool = False) -> None: self.file = None - self.fingerprints = set() + self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) @@ -39,26 +51,27 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprints.update(x.rstrip() for x in self.file) @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV: debug = settings.getbool('DUPEFILTER_DEBUG') return cls(job_dir(settings), debug) - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: return True self.fingerprints.add(fp) if self.file: self.file.write(fp + '\n') + return False - def request_fingerprint(self, request): + def request_fingerprint(self, request: Request) -> str: return request_fingerprint(request) - def close(self, reason): + def close(self, reason: str) -> None: if self.file: self.file.close() - def log(self, request, spider): + def log(self, request: Request, spider: Spider) -> None: if self.debug: msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" args = {'request': request, 'referer': referer_str(request)} diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 66736b42f..541368423 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -24,7 +24,7 @@ def request_fingerprint( request: Request, include_headers: Optional[Iterable[Union[bytes, str]]] = None, keep_fragments: bool = False, -): +) -> str: """ Return the request fingerprint. From 68379197986ae3deb81a545b5fd6920ea3347094 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 26 Apr 2021 14:55:02 -0300 Subject: [PATCH 1710/1853] Add peek method to queues (#5112) --- pylintrc | 1 + scrapy/pqueues.py | 59 +++++++--- scrapy/squeues.py | 59 +++++++--- tests/test_pqueues.py | 144 +++++++++++++++++++++++ tests/test_squeues_request.py | 214 ++++++++++++++++++++++++++++++++++ 5 files changed, 447 insertions(+), 30 deletions(-) create mode 100644 tests/test_pqueues.py create mode 100644 tests/test_squeues_request.py diff --git a/pylintrc b/pylintrc index 5b6b9fab0..972bf99de 100644 --- a/pylintrc +++ b/pylintrc @@ -24,6 +24,7 @@ disable=abstract-method, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, + consider-using-with, cyclic-import, dangerous-default-value, deprecated-method, diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a9aa6c649..b4b63e7c7 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -3,6 +3,7 @@ import logging from scrapy.utils.misc import create_instance + logger = logging.getLogger(__name__) @@ -17,8 +18,7 @@ def _path_safe(text): >>> _path_safe('some@symbol?').startswith('some_symbol_') True """ - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' - for c in text]) + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() @@ -35,6 +35,9 @@ class ScrapyPriorityQueue: * close() * __len__() + Optionally, the queue could provide a ``peek`` method, that should return the + next object to be returned by ``pop``, but without removing it from the queue. + ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls argument, which is a class used to instantiate a new (internal) queue when a new priority is allocated. @@ -70,10 +73,12 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) def qfactory(self, key): - return create_instance(self.downstream_queue_cls, - None, - self.crawler, - self.key + '/' + str(key)) + return create_instance( + self.downstream_queue_cls, + None, + self.crawler, + self.key + '/' + str(key), + ) def priority(self, request): return -request.priority @@ -99,6 +104,18 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + if self.curprio is None: + return None + queue = self.queues[self.curprio] + return queue.peek() + def close(self): active = [] for p, q in self.queues.items(): @@ -116,8 +133,7 @@ class DownloaderInterface: self.downloader = crawler.engine.downloader def stats(self, possible_slots): - return [(self._active_downloads(slot), slot) - for slot in possible_slots] + return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request): return self.downloader._get_slot_key(request, None) @@ -162,10 +178,12 @@ class DownloaderAwarePriorityQueue: self.pqueues[slot] = self.pqfactory(slot, startprios) def pqfactory(self, slot, startprios=()): - return ScrapyPriorityQueue(self.crawler, - self.downstream_queue_cls, - self.key + '/' + _path_safe(slot), - startprios) + return ScrapyPriorityQueue( + self.crawler, + self.downstream_queue_cls, + self.key + '/' + _path_safe(slot), + startprios, + ) def pop(self): stats = self._downloader_interface.stats(self.pqueues) @@ -187,9 +205,22 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] queue.push(request) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + stats = self._downloader_interface.stats(self.pqueues) + if not stats: + return None + slot = min(stats)[1] + queue = self.pqueues[slot] + return queue.peek() + def close(self): - active = {slot: queue.close() - for slot, queue in self.pqueues.items()} + active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 77ffda6f7..44898ba08 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -19,7 +19,6 @@ def _with_mkdir(queue_class): dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -38,6 +37,20 @@ def _serializable_queue(queue_class, serialize, deserialize): if s: return deserialize(s) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + if s: + return deserialize(s) + return SerializableQueue @@ -59,12 +72,21 @@ def _scrapy_serialization_queue(queue_class): def pop(self): request = super().pop() - if not request: return None + return request_from_dict(request, self.spider) - request = request_from_dict(request, self.spider) - return request + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + request = super().peek() + if not request: + return None + return request_from_dict(request, self.spider) return ScrapyRequestQueue @@ -76,6 +98,19 @@ def _scrapy_non_serialization_queue(queue_class): def from_crawler(cls, crawler, *args, **kwargs): return cls() + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + return s + return ScrapyRequestQueue @@ -109,17 +144,9 @@ MarshalLifoDiskQueueNonRequest = _serializable_queue( marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue( - PickleFifoDiskQueueNonRequest -) -PickleLifoDiskQueue = _scrapy_serialization_queue( - PickleLifoDiskQueueNonRequest -) -MarshalFifoDiskQueue = _scrapy_serialization_queue( - MarshalFifoDiskQueueNonRequest -) -MarshalLifoDiskQueue = _scrapy_serialization_queue( - MarshalLifoDiskQueueNonRequest -) +PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) +PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) +MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) +MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py new file mode 100644 index 000000000..ec55033d1 --- /dev/null +++ b/tests/test_pqueues.py @@ -0,0 +1,144 @@ +import tempfile +import unittest + +import queuelib + +from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue +from scrapy.spiders import Spider +from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.test import get_crawler + +from tests.test_scheduler import MockDownloader, MockEngine + + +class PriorityQueueTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("foo") + + def test_queue_push_pop_one(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + queue.push(req1) + self.assertEqual(len(queue), 1) + dequeued = queue.pop() + self.assertEqual(len(queue), 0) + self.assertEqual(dequeued.url, req1.url) + self.assertEqual(dequeued.priority, req1.priority) + self.assertEqual(queue.close(), []) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + queue.peek() + queue.close() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertEqual(len(queue), 0) + self.assertIsNone(queue.peek()) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + self.assertEqual(queue.peek().url, req1.url) + self.assertEqual(queue.pop().url, req1.url) + self.assertEqual(len(queue), 2) + self.assertEqual(queue.peek().url, req2.url) + self.assertEqual(queue.pop().url, req2.url) + self.assertEqual(len(queue), 1) + self.assertEqual(queue.peek().url, req3.url) + self.assertEqual(queue.pop().url, req3.url) + self.assertEqual(queue.close(), []) + + def test_queue_push_pop_priorities(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + req2 = Request("https://example.org/2", priority=2) + req3 = Request("https://example.org/3", priority=3) + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + dequeued = queue.pop() + self.assertEqual(len(queue), 2) + self.assertEqual(dequeued.url, req3.url) + self.assertEqual(dequeued.priority, req3.priority) + self.assertEqual(queue.close(), [-1, -2]) + + +class DownloaderAwarePriorityQueueTest(unittest.TestCase): + def setUp(self): + crawler = get_crawler(Spider) + crawler.engine = MockEngine(downloader=MockDownloader()) + self.queue = DownloaderAwarePriorityQueue.from_crawler( + crawler=crawler, + downstream_queue_cls=FifoMemoryQueue, + key="foo/bar", + ) + + def tearDown(self): + self.queue.close() + + def test_push_pop(self): + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + self.queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + self.queue.peek() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + self.assertEqual(len(self.queue), 0) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.peek().url, req1.url) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.peek().url, req2.url) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.peek().url, req3.url) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertIsNone(self.queue.peek()) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py new file mode 100644 index 000000000..c5fcc1853 --- /dev/null +++ b/tests/test_squeues_request.py @@ -0,0 +1,214 @@ +import shutil +import tempfile +import unittest + +import queuelib + +from scrapy.squeues import ( + PickleFifoDiskQueue, + PickleLifoDiskQueue, + MarshalFifoDiskQueue, + MarshalLifoDiskQueue, + FifoMemoryQueue, + LifoMemoryQueue, +) +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +""" +Queues that handle requests +""" + + +class BaseQueueTestCase(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") + self.qpath = self.tempfilename() + self.qdir = self.mkdtemp() + self.crawler = get_crawler(Spider) + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def tempfilename(self): + with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: + return nf.name + + def mkdtemp(self): + return tempfile.mkdtemp(dir=self.tmpdir) + + +class RequestQueueTestMixin: + def queue(self): + raise NotImplementedError() + + def test_one_element_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req.url) + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_one_element_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class FifoQueueMixin(RequestQueueTestMixin): + def test_fifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_fifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class LifoQueueMixin(RequestQueueTestMixin): + def test_lifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_lifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + + +class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + + +class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo") + + +class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo") + + +class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return FifoMemoryQueue.from_crawler(crawler=self.crawler) + + +class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return LifoMemoryQueue.from_crawler(crawler=self.crawler) From ddea6b7bfa38bf5402d78350ab61e2c827ca49b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:16:14 -0300 Subject: [PATCH 1711/1853] Scheduler: minimal interface, API docs (#3559) --- docs/index.rst | 4 + docs/topics/architecture.rst | 5 +- docs/topics/scheduler.rst | 34 ++++ docs/topics/settings.rst | 3 +- scrapy/core/engine.py | 21 ++- scrapy/core/scheduler.py | 298 +++++++++++++++++++++++++++-------- scrapy/utils/job.py | 5 +- tests/test_scheduler_base.py | 159 +++++++++++++++++++ 8 files changed, 458 insertions(+), 71 deletions(-) create mode 100644 docs/topics/scheduler.rst create mode 100644 tests/test_scheduler_base.py diff --git a/docs/index.rst b/docs/index.rst index da264fb34..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -227,6 +227,7 @@ Extending Scrapy topics/extensions topics/api topics/signals + topics/scheduler topics/exporters @@ -248,6 +249,9 @@ Extending Scrapy :doc:`topics/signals` See all available signals and how to work with them. +:doc:`topics/scheduler` + Understand the scheduler component. + :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 074c59241..71d027c86 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The Scheduler receives requests from the engine and enqueues them for feeding -them later (also to the engine) when the engine requests them. +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine +requests them. .. _component-downloader: diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst new file mode 100644 index 000000000..57c24b76a --- /dev/null +++ b/docs/topics/scheduler.rst @@ -0,0 +1,34 @@ +.. _topics-scheduler: + +========= +Scheduler +========= + +.. module:: scrapy.core.scheduler + +The scheduler component receives requests from the :ref:`engine ` +and stores them into persistent and/or non-persistent data structures. +It also gets those requests and feeds them back to the engine when it +asks for a next request to be downloaded. + + +Overriding the default scheduler +================================ + +You can use your own custom scheduler class by supplying its full +Python path in the :setting:`SCHEDULER` setting. + + +Minimal scheduler interface +=========================== + +.. autoclass:: BaseScheduler + :members: + + +Default Scrapy scheduler +======================== + +.. autoclass:: Scheduler + :members: + :special-members: __len__ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1d5babcec..e4fb2baf7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1280,7 +1280,8 @@ SCHEDULER Default: ``'scrapy.core.scheduler.Scheduler'`` -The scheduler to use for crawling. +The scheduler class to be used for crawling. +See the :ref:`topics-scheduler` topic for details. .. setting:: SCHEDULER_DEBUG diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7a09bafa1..dd3225082 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,6 +17,7 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info from scrapy.utils.misc import create_instance, load_object @@ -73,12 +74,22 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(crawler.settings["SCHEDULER"]) + self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback + def _get_scheduler_class(self, settings: BaseSettings) -> type: + from scrapy.core.scheduler import BaseScheduler + scheduler_cls = load_object(settings["SCHEDULER"]) + if not issubclass(scheduler_cls, BaseScheduler): + raise TypeError( + f"The provided scheduler class ({settings['SCHEDULER']})" + " does not fully implement the scheduler interface" + ) + return scheduler_cls + @inlineCallbacks def start(self) -> Deferred: if self.running: @@ -301,7 +312,8 @@ class ExecutionEngine: start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - yield scheduler.open(spider) + if hasattr(scheduler, "open"): + yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) @@ -345,8 +357,9 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) - dfd.addErrback(log_failure('Scheduler close failure')) + if hasattr(self.slot.scheduler, "close"): + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( signal=signals.spider_closed, spider=spider, reason=reason, diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 9ce823dbc..5ba0fb63b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,42 +1,179 @@ -import os import json import logging -from os.path import join, exists +import os +from abc import abstractmethod +from os.path import exists, join +from typing import Optional, Type, TypeVar -from scrapy.utils.misc import load_object, create_instance +from twisted.internet.defer import Deferred + +from scrapy.crawler import Crawler +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.job import job_dir +from scrapy.utils.misc import create_instance, load_object logger = logging.getLogger(__name__) -class Scheduler: +class BaseSchedulerMeta(type): """ - Scrapy Scheduler. It allows to enqueue requests and then get - a next request to download. Scheduler is also handling duplication - filtering, via dupefilter. - - Prioritization and queueing is not performed by the Scheduler. - User sets ``priority`` field for each Request, and a PriorityQueue - (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities - to dequeue requests in a desired order. - - Scheduler uses two PriorityQueue instances, configured to work in-memory - and on-disk (optional). When on-disk queue is present, it is used by - default, and an in-memory queue is used as a fallback for cases where - a disk queue can't handle a request (can't serialize it). - - :setting:`SCHEDULER_MEMORY_QUEUE` and - :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes - which PriorityQueue instances would be instantiated with, to keep requests - on disk and in memory respectively. - - Overall, Scheduler is an object which holds several PriorityQueue instances - (in-memory and on-disk) and implements fallback logic for them. - Also, it handles dupefilters. + Metaclass to check scheduler classes against the necessary interface """ - def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, - logunser=False, stats=None, pqclass=None, crawler=None): + def __instancecheck__(cls, instance): + return cls.__subclasscheck__(type(instance)) + + def __subclasscheck__(cls, subclass): + return ( + hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) + and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request) + and hasattr(subclass, "next_request") and callable(subclass.next_request) + ) + + +class BaseScheduler(metaclass=BaseSchedulerMeta): + """ + The scheduler component is responsible for storing requests received from + the engine, and feeding them back upon request (also to the engine). + + The original sources of said requests are: + + * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods + * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods + + The order in which the scheduler returns its stored requests (via the ``next_request`` method) + plays a great part in determining the order in which those requests are downloaded. + + The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. + """ + + @classmethod + def from_crawler(cls, crawler: Crawler): + """ + Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. + """ + return cls() + + def open(self, spider: Spider) -> Optional[Deferred]: + """ + Called when the spider is opened by the engine. It receives the spider + instance as argument and it's useful to execute initialization code. + + :param spider: the spider object for the current crawl + :type spider: :class:`~scrapy.spiders.Spider` + """ + pass + + def close(self, reason: str) -> Optional[Deferred]: + """ + Called when the spider is closed by the engine. It receives the reason why the crawl + finished as argument and it's useful to execute cleaning code. + + :param reason: a string which describes the reason why the spider was closed + :type reason: :class:`str` + """ + pass + + @abstractmethod + def has_pending_requests(self) -> bool: + """ + ``True`` if the scheduler has enqueued requests, ``False`` otherwise + """ + raise NotImplementedError() + + @abstractmethod + def enqueue_request(self, request: Request) -> bool: + """ + Process a request received by the engine. + + Return ``True`` if the request is stored correctly, ``False`` otherwise. + + If ``False``, the engine will fire a ``request_dropped`` signal, and + will not make further attempts to schedule the request at a later time. + For reference, the default Scrapy scheduler returns ``False`` when the + request is rejected by the dupefilter. + """ + raise NotImplementedError() + + @abstractmethod + def next_request(self) -> Optional[Request]: + """ + Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + to indicate that there are no requests to be considered ready at the moment. + + Returning ``None`` implies that no request from the scheduler will be sent + to the downloader in the current reactor cycle. The engine will continue + calling ``next_request`` until ``has_pending_requests`` is ``False``. + """ + raise NotImplementedError() + + +SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") + + +class Scheduler(BaseScheduler): + """ + Default Scrapy scheduler. This implementation also handles duplication + filtering via the :setting:`dupefilter `. + + This scheduler stores requests into several priority queues (defined by the + :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues + are backed by either memory or disk based queues (respectively defined by the + :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). + + Request prioritization is almost entirely delegated to the priority queue. The only + prioritization performed by this scheduler is using the disk-based queue if present + (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based + queue if a serialization error occurs. If the disk queue is not present, the memory one + is used directly. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ + def __init__( + self, + dupefilter, + jobdir: Optional[str] = None, + dqclass=None, + mqclass=None, + logunser: bool = False, + stats=None, + pqclass=None, + crawler: Optional[Crawler] = None, + ): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.pqclass = pqclass @@ -47,34 +184,57 @@ class Scheduler: self.crawler = crawler @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = create_instance(dupefilter_cls, settings, crawler) - pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) - mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('SCHEDULER_DEBUG') - return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, - stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, - mqclass=mqclass, crawler=crawler) + def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV: + """ + Factory method, initializes the scheduler with arguments taken from the crawl settings + """ + dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS']) + return cls( + dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']), + mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']), + logunser=crawler.settings.getbool('SCHEDULER_DEBUG'), + stats=crawler.stats, + pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']), + crawler=crawler, + ) - def has_pending_requests(self): + def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider): + def open(self, spider: Spider) -> Optional[Deferred]: + """ + (1) initialize the memory queue + (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (3) return the result of the dupefilter's ``open`` method + """ self.spider = spider self.mqs = self._mq() self.dqs = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason): - if self.dqs: + def close(self, reason: str) -> Optional[Deferred]: + """ + (1) dump pending requests to disk if there is a disk queue + (2) return the result of the dupefilter's ``close`` method + """ + if self.dqs is not None: state = self.dqs.close() + assert isinstance(self.dqdir, str) self._write_dqs_state(self.dqdir, state) return self.df.close(reason) - def enqueue_request(self, request): + def enqueue_request(self, request: Request) -> bool: + """ + Unless the received request is filtered out by the Dupefilter, attempt to push + it into the disk queue, falling back to pushing it into the memory queue. + + Increment the appropriate stats, such as: ``scheduler/enqueued``, + ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + + Return ``True`` if the request was stored successfully, ``False`` otherwise. + """ if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False @@ -87,24 +247,35 @@ class Scheduler: self.stats.inc_value('scheduler/enqueued', spider=self.spider) return True - def next_request(self): + def next_request(self) -> Optional[Request]: + """ + Return a :class:`~scrapy.http.Request` object from the memory queue, + falling back to the disk queue if the memory queue is empty. + Return ``None`` if there are no more enqueued requests. + + Increment the appropriate stats, such as: ``scheduler/dequeued``, + ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + """ request = self.mqs.pop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: request = self._dqpop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider) - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued', spider=self.spider) return request - def __len__(self): - return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs) + def __len__(self) -> int: + """ + Return the total amount of enqueued requests + """ + return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs) - def _dqpush(self, request): + def _dqpush(self, request: Request) -> bool: if self.dqs is None: - return + return False try: self.dqs.push(request) except ValueError as e: # non serializable request @@ -115,18 +286,18 @@ class Scheduler: logger.warning(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) self.logunser = False - self.stats.inc_value('scheduler/unserializable', - spider=self.spider) - return + self.stats.inc_value('scheduler/unserializable', spider=self.spider) + return False else: return True - def _mqpush(self, request): + def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self): - if self.dqs: + def _dqpop(self) -> Optional[Request]: + if self.dqs is not None: return self.dqs.pop() + return None def _mq(self): """ Create a new priority queue instance, with in-memory storage """ @@ -150,21 +321,22 @@ class Scheduler: {'queuesize': len(q)}, extra={'spider': self.spider}) return q - def _dqdir(self, jobdir): + def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """ Return a folder name to keep disk queue state at """ - if jobdir: + if jobdir is not None: dqdir = join(jobdir, 'requests.queue') if not exists(dqdir): os.makedirs(dqdir) return dqdir + return None - def _read_dqs_state(self, dqdir): + def _read_dqs_state(self, dqdir: str) -> list: path = join(dqdir, 'active.json') if not exists(path): - return () + return [] with open(path) as f: return json.load(f) - def _write_dqs_state(self, dqdir, state): + def _write_dqs_state(self, dqdir: str, state: list) -> None: with open(join(dqdir, 'active.json'), 'w') as f: json.dump(state, f) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 4f1e601fc..c92ef36f5 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ import os +from typing import Optional + +from scrapy.settings import BaseSettings -def job_dir(settings): +def job_dir(settings: BaseSettings) -> Optional[str]: path = settings['JOBDIR'] if path and not os.path.exists(path): os.makedirs(path) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py new file mode 100644 index 000000000..bf90b4320 --- /dev/null +++ b/tests/test_scheduler_base.py @@ -0,0 +1,159 @@ +from typing import Dict, Optional +from unittest import TestCase +from urllib.parse import urljoin, urlparse + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase as TwistedTestCase + +from scrapy.core.scheduler import BaseScheduler +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.request import request_fingerprint + +from tests.mockserver import MockServer + + +PATHS = ["/a", "/b", "/c"] +URLS = [urljoin("https://example.org", p) for p in PATHS] + + +class MinimalScheduler: + def __init__(self) -> None: + self.requests: Dict[str, Request] = {} + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def enqueue_request(self, request: Request) -> bool: + fp = request_fingerprint(request) + if fp not in self.requests: + self.requests[fp] = request + return True + return False + + def next_request(self) -> Optional[Request]: + if self.has_pending_requests(): + fp, request = self.requests.popitem() + return request + return None + + +class SimpleScheduler(MinimalScheduler): + def open(self, spider: Spider) -> defer.Deferred: + return defer.succeed("open") + + def close(self, reason: str) -> defer.Deferred: + return defer.succeed("close") + + def __len__(self) -> int: + return len(self.requests) + + +class TestSpider(Spider): + name = "test" + + def __init__(self, mockserver, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_urls = map(mockserver.url, PATHS) + + def parse(self, response): + return {"path": urlparse(response.url).path} + + +class InterfaceCheckMixin: + def test_scheduler_class(self): + self.assertTrue(isinstance(self.scheduler, BaseScheduler)) + self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + + +class BaseSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = BaseScheduler() + + def test_methods(self): + self.assertIsNone(self.scheduler.open(Spider("foo"))) + self.assertIsNone(self.scheduler.close("finished")) + self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests) + self.assertRaises(NotImplementedError, self.scheduler.enqueue_request, Request("https://example.org")) + self.assertRaises(NotImplementedError, self.scheduler.next_request) + + +class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = MinimalScheduler() + + def test_open_close(self): + with self.assertRaises(AttributeError): + self.scheduler.open(Spider("foo")) + with self.assertRaises(AttributeError): + self.scheduler.close("finished") + + def test_len(self): + with self.assertRaises(AttributeError): + self.scheduler.__len__() + with self.assertRaises(TypeError): + len(self.scheduler) + + def test_enqueue_dequeue(self): + self.assertFalse(self.scheduler.has_pending_requests()) + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + self.assertTrue(self.scheduler.has_pending_requests) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + self.assertFalse(self.scheduler.has_pending_requests()) + + +class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = SimpleScheduler() + + @defer.inlineCallbacks + def test_enqueue_dequeue(self): + open_result = yield self.scheduler.open(Spider("foo")) + self.assertEqual(open_result, "open") + self.assertFalse(self.scheduler.has_pending_requests()) + + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(URLS)) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + close_result = yield self.scheduler.close("") + self.assertEqual(close_result, "close") + + +class MinimalSchedulerCrawlTest(TwistedTestCase): + scheduler_cls = MinimalScheduler + + @defer.inlineCallbacks + def test_crawl(self): + with MockServer() as mockserver: + settings = {"SCHEDULER": self.scheduler_cls} + with LogCapture() as log: + yield CrawlerRunner(settings).crawl(TestSpider, mockserver) + for path in PATHS: + self.assertIn(f"{{'path': '{path}'}}", str(log)) + self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + + +class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): + scheduler_cls = SimpleScheduler From 02ae1deaf499e79dc8ececf4d5dcea6daef0ade0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 27 Apr 2021 09:41:44 -0300 Subject: [PATCH 1712/1853] Deprecate unused squeues (#5117) --- scrapy/squeues.py | 47 +++++++++++++++++++++++++++++++++++-------- tests/test_squeues.py | 16 +++++++-------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 44898ba08..16f7bf4b6 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -8,6 +8,7 @@ import pickle from queuelib import queue +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.reqser import request_to_dict, request_from_dict @@ -123,30 +124,60 @@ def _pickle_serialize(obj): raise ValueError(str(e)) from e -PickleFifoDiskQueueNonRequest = _serializable_queue( +_PickleFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads ) -PickleLifoDiskQueueNonRequest = _serializable_queue( +_PickleLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads ) -MarshalFifoDiskQueueNonRequest = _serializable_queue( +_MarshalFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads ) -MarshalLifoDiskQueueNonRequest = _serializable_queue( +_MarshalLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) -PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) -MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) -MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) +# public queue classes +PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) +PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) +MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) +MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) + + +# deprecated queue classes +_subclass_warn_message = "{cls} inherits from deprecated class {old}" +_instance_warn_message = "{cls} is deprecated" +PickleFifoDiskQueueNonRequest = create_deprecated_class( + name="PickleFifoDiskQueueNonRequest", + new_class=_PickleFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +PickleLifoDiskQueueNonRequest = create_deprecated_class( + name="PickleLifoDiskQueueNonRequest", + new_class=_PickleLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalFifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalFifoDiskQueueNonRequest", + new_class=_MarshalFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalLifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalLifoDiskQueueNonRequest", + new_class=_MarshalLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index becacce62..acc821b83 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -3,10 +3,10 @@ import sys from queuelib.tests import test_queue as t from scrapy.squeues import ( - MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, - MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, - PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, - PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue + _MarshalFifoSerializationDiskQueue, + _MarshalLifoSerializationDiskQueue, + _PickleFifoSerializationDiskQueue, + _PickleLifoSerializationDiskQueue, ) from scrapy.item import Item, Field from scrapy.http import Request @@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): @@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) def test_serialize_item(self): q = self.queue() @@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin: class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return MarshalLifoDiskQueue(self.qpath) + return _MarshalLifoSerializationDiskQueue(self.qpath) class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return PickleLifoDiskQueue(self.qpath) + return _PickleLifoSerializationDiskQueue(self.qpath) def test_serialize_item(self): q = self.queue() From 4f500342c8ad4674b191e1fab0d1b2ac944d7d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 28 Apr 2021 11:57:44 +0200 Subject: [PATCH 1713/1853] Require setuptools, scrapy/cmdline.py, /setup.py and tests/test_webclient.py import pkg_resources --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2b60a10af..b1bb64575 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.0,<4.0', + 'setuptools', ] extras_require = {} cpython_dependencies = [ From 19c7415aae1678631d5ca115a13094b6bd70f245 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 1 May 2021 16:34:39 -0300 Subject: [PATCH 1714/1853] Request type hints --- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/http/request/__init__.py | 69 ++++++++++++++++----------- scrapy/utils/curl.py | 2 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5965a1c6c..f1fdc3858 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -98,7 +98,7 @@ def get_retry_request( {'request': request, 'retry_times': retry_times, 'reason': reason}, extra={'spider': spider} ) - new_request = request.copy() + new_request: Request = request.copy() new_request.meta['retry_times'] = retry_times new_request.dont_filter = True if priority_adjust is None: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..3cce9f501 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,22 +4,38 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from typing import Callable, List, Optional, Type, TypeVar, Union + from w3lib.url import safe_url_string +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs + + +RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class Request(object_ref): - - def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, cb_kwargs=None): - + def __init__( + self, + url: str, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Optional[dict] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]]=None, + meta: Optional[dict] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict] = None, + ) -> None: self._encoding = encoding # this one has to be set first self.method = str(method).upper() self._set_url(url) @@ -44,23 +60,23 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) @property - def cb_kwargs(self): + def cb_kwargs(self) -> dict: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self): + def meta(self) -> dict: if self._meta is None: self._meta = {} return self._meta - def _get_url(self): + def _get_url(self) -> str: return self._url - def _set_url(self, url): + def _set_url(self, url: str) -> None: if not isinstance(url, str): - raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') + raise TypeError(f"Request url must be str, got {type(url).__name__}") s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) @@ -74,34 +90,28 @@ class Request(object_ref): url = property(_get_url, obsolete_setter(_set_url, 'url')) - def _get_body(self): + def _get_body(self) -> bytes: return self._body - def _set_body(self, body): - if body is None: - self._body = b'' - else: - self._body = to_bytes(body, self.encoding) + def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + self._body = b"" if body is None else to_bytes(body, self.encoding) body = property(_get_body, obsolete_setter(_set_body, 'body')) @property - def encoding(self): + def encoding(self) -> str: return self._encoding - def __str__(self): + def __str__(self) -> str: return f"<{self.method} {self.url}>" __repr__ = __str__ - def copy(self): - """Return a copy of this Request""" + def copy(self) -> RequestTypeVar: return self.replace() - def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ + def replace(self, *args, **kwargs) -> RequestTypeVar: + """Create a new Request with the same attributes except for those given new values""" for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) @@ -109,7 +119,9 @@ class Request(object_ref): return cls(*args, **kwargs) @classmethod - def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): + def from_curl( + cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs + ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -136,8 +148,7 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index d8b3deaa1..74f82ad75 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -54,7 +54,7 @@ def _parse_headers_and_cookies(parsed_args): return headers, cookies -def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): +def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command From 34b216289c31c27ad6256ff95382505a9d84adb3 Mon Sep 17 00:00:00 2001 From: Renne Rocha Date: Thu, 6 May 2021 11:34:05 -0300 Subject: [PATCH 1715/1853] Update link for reasoning value of URLLENGTH_LIMIT (#5134) --- docs/topics/settings.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e4fb2baf7..2506497e2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1619,7 +1619,7 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -1642,7 +1642,6 @@ case to see how to enable and use them. .. settingslist:: - .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search From cec36a9284641bb7b69a2081ad92cd7d4ee25934 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 10 May 2021 13:00:08 -0300 Subject: [PATCH 1716/1853] Refactor request to/from dict (#5130) --- docs/topics/request-response.rst | 17 ++- scrapy/http/request/__init__.py | 70 ++++++++++-- scrapy/http/request/json_request.py | 8 ++ scrapy/squeues.py | 8 +- scrapy/utils/reqser.py | 103 +++--------------- scrapy/utils/request.py | 27 ++++- ...t_utils_reqser.py => test_request_dict.py} | 76 +++++++++---- 7 files changed, 183 insertions(+), 126 deletions(-) rename tests/{test_utils_reqser.py => test_request_dict.py} (68%) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 500781c05..73b5a858f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -26,10 +26,6 @@ Request objects .. autoclass:: Request - A :class:`Request` object represents an HTTP request, which is usually - generated in the Spider and executed by the Downloader, and thus generating - a :class:`Response`. - :param url: the URL of this request If the URL is invalid, a :exc:`ValueError` exception is raised. @@ -205,6 +201,8 @@ Request objects ``failure.request.cb_kwargs`` in the request's errback. For more information, see :ref:`errback-cb_kwargs`. + .. autoattribute:: Request.attributes + .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: @@ -220,6 +218,15 @@ Request objects .. automethod:: from_curl + .. automethod:: to_dict + + +Other functions related to requests +----------------------------------- + +.. autofunction:: scrapy.utils.request.request_from_dict + + .. _topics-request-response-ref-request-callback-arguments: Passing additional data to callback functions @@ -642,6 +649,8 @@ dealing with JSON requests. data into JSON format. :type dumps_kwargs: dict + .. autoattribute:: JsonRequest.attributes + JsonRequest usage example ------------------------- diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..ad884feac 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,17 +4,37 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +import inspect +from typing import Optional, Tuple + from w3lib.url import safe_url_string +import scrapy +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs class Request(object_ref): + """Represents an HTTP request, which is usually generated in a Spider and + executed by the Downloader, thus generating a :class:`Response`. + """ + + attributes: Tuple[str, ...] = ( + "url", "callback", "method", "headers", "body", + "cookies", "meta", "encoding", "priority", + "dont_filter", "errback", "flags", "cb_kwargs", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + + Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + :func:`~scrapy.utils.request.request_from_dict`. + """ def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, @@ -99,11 +119,8 @@ class Request(object_ref): return self.replace() def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ - for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: + """Create a new Request with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) @@ -136,8 +153,43 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) + + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + """Return a dictionary containing the Request's data. + + Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. + + If a spider is given, this method will try to find out the name of the spider methods used as callback + and errback and include them in the output dict, raising an exception if they cannot be found. + """ + d = { + "url": self.url, # urls are safe (safe_string_url) + "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback, + "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback, + "headers": dict(self.headers), + } + for attr in self.attributes: + d.setdefault(attr, getattr(self, attr)) + if type(self) is not Request: + d["_class"] = self.__module__ + '.' + self.__class__.__name__ + return d + + +def _find_method(obj, func): + """Helper function for Request.to_dict""" + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original function object because instance + # method objects are generated each time attribute is retrieved from instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError(f"Function {func} is not an instance method in: {obj}") diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index eae3f9f6b..04e80d897 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings +from typing import Tuple from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): + + attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + def __init__(self, *args, **kwargs): dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) dumps_kwargs.setdefault('sort_keys', True) @@ -36,6 +40,10 @@ class JsonRequest(Request): self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + @property + def dumps_kwargs(self): + return self._dumps_kwargs + def replace(self, *args, **kwargs): body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 16f7bf4b6..dff9b1350 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -9,7 +9,7 @@ import pickle from queuelib import queue from scrapy.utils.deprecate import create_deprecated_class -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.request import request_from_dict def _with_mkdir(queue_class): @@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class): return cls(crawler, key) def push(self, request): - request = request_to_dict(request, self.spider) + request = request.to_dict(spider=self.spider) return super().push(request) def pop(self): request = super().pop() if not request: return None - return request_from_dict(request, self.spider) + return request_from_dict(request, spider=self.spider) def peek(self): """Returns the next object to be returned by :meth:`pop`, @@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class): request = super().peek() if not request: return None - return request_from_dict(request, self.spider) + return request_from_dict(request, spider=self.spider) return ScrapyRequestQueue diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d38b1bc4d..c254b9f82 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,95 +1,22 @@ -""" -Helper functions for serializing (and deserializing) requests. -""" -import inspect +import warnings +from typing import Optional -from scrapy.http import Request -from scrapy.utils.python import to_unicode -from scrapy.utils.misc import load_object +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.request import request_from_dict as _from_dict -def request_to_dict(request, spider=None): - """Convert Request object to a dict. - - If a spider is given, it will try to find out the name of the spider method - used in the callback and store that as the callback. - """ - cb = request.callback - if callable(cb): - cb = _find_method(spider, cb) - eb = request.errback - if callable(eb): - eb = _find_method(spider, eb) - d = { - 'url': to_unicode(request.url), # urls should be safe (safe_string_url) - 'callback': cb, - 'errback': eb, - 'method': request.method, - 'headers': dict(request.headers), - 'body': request.body, - 'cookies': request.cookies, - 'meta': request.meta, - '_encoding': request._encoding, - 'priority': request.priority, - 'dont_filter': request.dont_filter, - 'flags': request.flags, - 'cb_kwargs': request.cb_kwargs, - } - if type(request) is not Request: - d['_class'] = request.__module__ + '.' + request.__class__.__name__ - return d +warnings.warn( + ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead"), + category=ScrapyDeprecationWarning, + stacklevel=2, +) -def request_from_dict(d, spider=None): - """Create Request object from a dict. - - If a spider is given, it will try to resolve the callbacks looking at the - spider for methods with the same name. - """ - cb = d['callback'] - if cb and spider: - cb = _get_method(spider, cb) - eb = d['errback'] - if eb and spider: - eb = _get_method(spider, eb) - request_cls = load_object(d['_class']) if '_class' in d else Request - return request_cls( - url=to_unicode(d['url']), - callback=cb, - errback=eb, - method=d['method'], - headers=d['headers'], - body=d['body'], - cookies=d['cookies'], - meta=d['meta'], - encoding=d['_encoding'], - priority=d['priority'], - dont_filter=d['dont_filter'], - flags=d.get('flags'), - cb_kwargs=d.get('cb_kwargs'), - ) +def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict: + return request.to_dict(spider=spider) -def _find_method(obj, func): - # Only instance methods contain ``__func__`` - if obj and hasattr(func, '__func__'): - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError(f"Function {func} is not an instance method in: {obj}") - - -def _get_method(obj, name): - name = str(name) - try: - return getattr(obj, name) - except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") +def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request": + return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 541368423..57dcc5f2c 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url -from scrapy.http import Request +from scrapy import Request, Spider from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]: if referrer is None: return referrer return to_unicode(referrer, errors='replace') + + +def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: + """Create a :class:`~scrapy.Request` object from a dict. + + If a spider is given, it will try to resolve the callbacks looking at the + spider for methods with the same name. + """ + request_cls = load_object(d["_class"]) if "_class" in d else Request + kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} + if d.get("callback") and spider: + kwargs["callback"] = _get_method(spider, d["callback"]) + if d.get("errback") and spider: + kwargs["errback"] = _get_method(spider, d["errback"]) + return request_cls(**kwargs) + + +def _get_method(obj, name): + """Helper function for request_from_dict""" + name = str(name) + try: + return getattr(obj, name) + except AttributeError: + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.py similarity index 68% rename from tests/test_utils_reqser.py rename to tests/test_request_dict.py index ee68cf6b1..5bdcb975b 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_request_dict.py @@ -1,8 +1,16 @@ +import sys import unittest +import warnings +from contextlib import suppress -from scrapy.http import Request, FormRequest -from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy import Spider, Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import FormRequest, JsonRequest +from scrapy.utils.request import request_from_dict + + +class CustomRequest(Request): + pass class RequestSerializationTest(unittest.TestCase): @@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase): priority=20, meta={'a': 'b'}, cb_kwargs={'k': 'v'}, - flags=['testFlag']) + flags=['testFlag'], + ) self._assert_serializes_ok(r, spider=self.spider) def test_latin1_body(self): @@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def _assert_serializes_ok(self, request, spider=None): - d = request_to_dict(request, spider=spider) + d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) @@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) self.assertEqual(r1.flags, r2.flags) + if isinstance(r1, JsonRequest): + self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) def test_request_class(self): - r = FormRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) - r = CustomRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) + r1 = FormRequest("http://www.example.com") + self._assert_serializes_ok(r1, spider=self.spider) + r2 = CustomRequest("http://www.example.com") + self._assert_serializes_ok(r2, spider=self.spider) + r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) + self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider.parse_item, @@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider.parse_item_reference, errback=self.spider.handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], 'parse_item_reference') self.assertEqual(request_dict['errback'], 'handle_error_reference') @@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider._TestSpider__parse_item_reference, errback=self.spider._TestSpider__handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], '_TestSpider__parse_item_reference') self.assertEqual(request_dict['errback'], @@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, request_to_dict, r) - self.assertRaises(ValueError, request_to_dict, r, spider=self.spider) + self.assertRaises(ValueError, r.to_dict, spider=self.spider) def test_unserializable_callback2(self): r = Request("http://www.example.com", callback=self.spider.parse_item) - self.assertRaises(ValueError, request_to_dict, r) + self.assertRaises(ValueError, r.to_dict, spider=None) def test_unserializable_callback3(self): """Parser method is removed or replaced dynamically.""" class MySpider(Spider): - name = 'my_spider' def parse(self, response): @@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) setattr(spider, 'parse', None) - self.assertRaises(ValueError, request_to_dict, r, spider=spider) + self.assertRaises(ValueError, r.to_dict, spider=spider) + + def test_callback_not_available(self): + """Callback method is not available in the spider passed to from_dict""" + spider = TestSpiderDelegation() + r = Request("http://www.example.com", callback=spider.delegated_callback) + d = r.to_dict(spider=spider) + self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) + + +class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): + def _assert_serializes_ok(self, request, spider=None): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with suppress(KeyError): + del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning + + from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict + + request_copy = _from_dict(_to_dict(request, spider), spider) + self._assert_same_request(request, request_copy) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead", + str(caught[0].message), + ) class TestSpiderMixin: @@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin): def __parse_item_private(self, response): pass - - -class CustomRequest(Request): - pass From bd60c3f41fd25e7b5a413cf5c112166b813c2691 Mon Sep 17 00:00:00 2001 From: Shinichi Takayanagi Date: Tue, 11 May 2021 04:58:04 +0900 Subject: [PATCH 1717/1853] More documentation for setting spider atributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17 * Update feed-exports.rst * Update feed-exports.rst * Reflects the comments * Remove redundant newline * Update docs/topics/feed-exports.rst Co-authored-by: Adrián Chaves * Apply suggestions from code review Co-authored-by: Adrián Chaves Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- docs/topics/feed-exports.rst | 3 +++ docs/topics/spiders.rst | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e772a461c..26c247cdd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -135,6 +135,9 @@ Here are some examples to illustrate: - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` +.. note:: :ref:`Spider arguments ` become spider attributes, hence + they can also be used as storage URI parameters. + .. _topics-feed-storage-backends: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 2056664c7..a3e9f410f 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -294,6 +294,14 @@ The above example can also be written as follows:: def start_requests(self): yield scrapy.Request(f'http://www.example.com/categories/{self.category}') +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling +:class:`CrawlerProcess.crawl ` or +:class:`CrawlerRunner.crawl `:: + + process = CrawlerProcess() + process.crawl(MySpider, category="electronics") + Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. If you were to set the ``start_urls`` attribute from the command line, From c5b1ee810167266fcd259f263dbfc0fe0204761a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 May 2021 09:04:53 +0200 Subject: [PATCH 1718/1853] Make Twisted[http2] installation optional (#5113) Co-authored-by: Eugenio Lacuesta --- conftest.py | 9 +++++++++ docs/topics/settings.rst | 14 ++++++++----- setup.py | 3 +-- tests/test_downloader_handlers_http2.py | 26 ++++++++++++++++++++----- tests/test_http2_client_protocol.py | 13 ++++++++----- tox.ini | 11 +++++------ 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/conftest.py b/conftest.py index e4dd80de0..4931c5a79 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'): if file_path and file_path[0] != '#': collect_ignore.append(file_path) +if not H2_ENABLED: + collect_ignore.extend( + ( + 'scrapy/core/downloader/handlers/http2.py', + *_py_files("scrapy/core/http2"), + ) + ) + @pytest.fixture() def chdir(tmpdir): diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2506497e2..0b290598f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``:: .. _http2: -The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update -:setting:`DOWNLOAD_HANDLERS` as follows:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2: - DOWNLOAD_HANDLERS = { - 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', - } +#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to + enable HTTP/2 support in Twisted. + +#. Update :setting:`DOWNLOAD_HANDLERS` as follows:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } .. warning:: diff --git a/setup.py b/setup.py index b1bb64575..ed2b6e347 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted[http2]>=17.9.0', + 'Twisted>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', @@ -31,7 +31,6 @@ install_requires = [ 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'h2>=3.0,<4.0', 'setuptools', ] extras_require = {} diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 439778014..53bb4fe92 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,5 +1,5 @@ import json -from unittest import mock +from unittest import mock, skipIf from pytest import mark from testfixtures import LogCapture @@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server from twisted.web.error import SchemeNotSupported +from twisted.web.http import H2_ENABLED -from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import create_instance @@ -21,11 +21,17 @@ from tests.test_downloader_handlers import ( ) +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): + scheme = 'https' - download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + def test_protocol(self): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) @@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase): super(Https2InvalidDNSPattern, self).setUp() +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2CustomCiphers(Https11CustomCiphers): scheme = 'https' - download_handler_cls = H2DownloadHandler + + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler class Http2MockServerTestCase(Http11MockServerTestCase): @@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase): } +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ProxyTestCase(Http11ProxyTestCase): # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase): scheme = 'https' host = u'127.0.0.1' - download_handler_cls = H2DownloadHandler expected_http_proxy_request_body = b'/' + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8b2f6a11d..677ede92b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,10 +5,9 @@ import re import shutil import string from ipaddress import IPv4Address -from unittest import mock +from unittest import mock, skipIf from urllib.parse import urlencode -from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint @@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import ResponseFailed, URI -from twisted.web.http import Request as TxRequest +from twisted.web.http import H2_ENABLED, Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings from scrapy.spiders import Spider @@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: return PrivateCertificate.loadPEM(pem) +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): scheme = 'https' key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') @@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase): uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_failure(failure: Failure): self.assertTrue(len(failure.value.reasons) > 0) + from h2.exceptions import InvalidBodyLengthError self.assertTrue(any( isinstance(error, InvalidBodyLengthError) for error in failure.value.reasons @@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_inactive_stream(failure): self.assertIsNotNone(failure.check(ResponseFailed)) + from scrapy.core.http2.stream import InactiveStreamClosed self.assertTrue(any( isinstance(e, InactiveStreamClosed) for e in failure.value.reasons @@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url) def assert_invalid_hostname(failure: Failure): + from scrapy.core.http2.stream import InvalidHostname self.assertIsNotNone(failure.check(InvalidHostname)) error_msg = str(failure.value) self.assertIn('localhost', error_msg) @@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_timeout_error(failure: Failure): for err in failure.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol if isinstance(err, TimeoutError): self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) break diff --git a/tox.ini b/tox.ini index 5b0606f8f..8167aff96 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,8 @@ commands = basepython = python3 deps = {[testenv]deps} + # Twisted[http2] is required to import some files + Twisted[http2]>=17.9.0 pytest-flake8 commands = py.test --flake8 {posargs:docs scrapy tests} @@ -57,12 +59,7 @@ commands = [testenv:pylint] basepython = python3 deps = - {[testenv]deps} - # Optional dependencies - boto - reppy - robotexclusionrulesparser - # Test dependencies + {[testenv:extra-deps]deps} pylint commands = pylint conftest.py docs extras scrapy setup.py tests @@ -119,9 +116,11 @@ setenv = [testenv:extra-deps] deps = {[testenv]deps} + boto reppy robotexclusionrulesparser Pillow>=4.0.0 + Twisted[http2]>=17.9.0 [testenv:asyncio] commands = From ee682af3b06d48815dbdaa27c1177b94aaf679e1 Mon Sep 17 00:00:00 2001 From: Bhavesh <35660861+Bhavesh0327@users.noreply.github.com> Date: Wed, 12 May 2021 01:53:02 +0530 Subject: [PATCH 1719/1853] [Fix] Change the truncation limit of Proxy TunnelError from 32 to 1000 (#5007) * [Fix] Change the truncation limit oof Proxy TunnelError from 32 to 64 * [Fix] Change the truncation limit for Proxy tunnel error * [Fix] flake8 check * [Fix] formatting issues * [Remove] coverage report * [Fix] truncation error issue * [Fix] formatting issues * [Remove] coverage report --- scrapy/core/downloader/handlers/http11.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 25cb3ec62..073f35891 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -98,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): with this endpoint comes from the pool and a CONNECT has already been issued for it. """ - - _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') + _truncatedLength = 1000 + _responseAnswer = r'HTTP/1\.. (?P\d{3})(?P.{,' + str(_truncatedLength) + r'})' + _responseMatcher = re.compile(_responseAnswer.encode()) def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf @@ -144,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): extra = {'status': int(respm.group('status')), 'reason': respm.group('reason').strip()} else: - extra = rcvd_bytes[:32] + extra = rcvd_bytes[:self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError('Could not open CONNECT tunnel with proxy ' f'{self._host}:{self._port} [{extra!r}]') From 23cfdb058e80a18ee2b66e1b966355f3aca426d0 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 28 May 2021 09:45:06 +0000 Subject: [PATCH 1720/1853] Reducing amount of warnings during test run (#5162) * put flake8 options into separate file to remove pytest warnings * remove ResourceLeaked warning in pypy * suppress warnings from twisted * ignore deprecation warnings here * ignore deprecation warning in tests of deprecated methods * ignore deprecation warnings here * update test classes * don`t use deprecated method call * ignore deprecation warnings here * proper warning class * more selective ignoring * Revert "don`t use deprecated method call" This reverts commit 59216ab5603c4b47574382768614ef4c39d36747. --- .flake8 | 19 +++++++++++++++++++ .gitignore | 2 ++ conftest.py | 9 +++++---- pytest.ini | 19 ++----------------- tests/test_exporters.py | 12 ++++++++---- tests/test_feedexport.py | 4 ++-- tests/test_http_response.py | 12 ++++++++---- tests/test_item.py | 22 ++++++++++++---------- tests/test_utils_deprecate.py | 2 +- tests/test_utils_python.py | 9 +++++++-- 10 files changed, 66 insertions(+), 44 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..1c503fb0b --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] + +max-line-length = 119 +ignore = W503 + +exclude = +# Exclude files that are meant to provide top-level imports +# E402: Module level import not at top of file +# F401: Module imported but unused + scrapy/__init__.py E402 + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 + scrapy/spiders/__init__.py E402 F401 + + # Issues pending a review: + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 diff --git a/.gitignore b/.gitignore index 795e2605e..d77d24624 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ htmlcov/ .coverage .pytest_cache/ .coverage.* +coverage.* +test-output.* .cache/ .mypy_cache/ /tests/keys/localhost.crt diff --git a/conftest.py b/conftest.py index 4931c5a79..05b4ccdad 100644 --- a/conftest.py +++ b/conftest.py @@ -21,10 +21,11 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -for line in open('tests/ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +with open('tests/ignores.txt') as reader: + for line in reader: + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) if not H2_ENABLED: collect_ignore.extend( diff --git a/pytest.ini b/pytest.ini index 0aae09ff5..6de08c78d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,20 +20,5 @@ addopts = --ignore=docs/utils markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed -flake8-max-line-length = 119 -flake8-ignore = - W503 - - # Exclude files that are meant to provide top-level imports - # E402: Module level import not at top of file - # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 - - # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 +filterwarnings= + ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/tests/test_exporters.py b/tests/test_exporters.py index ebc477e74..04bae31d3 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -6,12 +6,14 @@ import tempfile import unittest from io import BytesIO from datetime import datetime +from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, @@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): - exporter = PythonItemExporter(binary=True) - value = self.item_class(name='John\xa3', age='22') - expected = {b'name': b'John\xc2\xa3', b'age': b'22'} - self.assertEqual(expected, exporter.export_item(value)) + with catch_warnings(): + filterwarnings('ignore', category=ScrapyDeprecationWarning) + exporter = PythonItemExporter(binary=True) + value = self.item_class(name='John\xa3', age='22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d248824fc..df7ec4461 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): - def __init__(self, uri): + def __init__(self, uri, *args, feed_options=None): self.path = file_uri_to_path(uri) def _store_in_thread(self, file): @@ -541,7 +541,7 @@ class LogOnStoreFileStorage: It can be used to make sure `store` method is invoked. """ - def __init__(self, uri): + def __init__(self, uri, feed_options=None): self.path = file_uri_to_path(uri) self.logger = getLogger() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f831ef5dc..04a594d03 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding @@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - self.assertEqual(response.body_as_unicode(), body_unicode) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest): r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertTrue(isinstance(r1.body_as_unicode(), str)) + self.assertEqual(r1.body_as_unicode(), unicode_string) # check response.text self.assertTrue(isinstance(r1.text, str)) diff --git a/tests/test_item.py b/tests/test_item.py index 78d204e34..c94bb44af 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta @@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase): class SubclassedItem(Item): pass - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertTrue(isinstance(BaseItem(), BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) + self.assertTrue(isinstance(Item(), BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) + # make sure internal checks using private _BaseItem class succeed + self.assertTrue(isinstance(BaseItem(), _BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) + self.assertTrue(isinstance(Item(), _BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) def test_deprecation_warning(self): """ diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 35d35b45d..e47afa266 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): # ignore subclassing warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) + warnings.simplefilter('ignore', MyWarning) class UserClass(Deprecated): pass diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3115cc92f..4b3964154 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,8 +5,9 @@ import platform import unittest from datetime import datetime from itertools import count -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, get_func_args, to_bytes, to_unicode, @@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase): pass _values = count() - wk = WeakKeyCache(lambda k: next(_values)) + + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() v = wk[k] self.assertEqual(v, wk[k]) From 216dd37953112e360348e19eaf8cae45a52fe87a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:16:40 -0300 Subject: [PATCH 1721/1853] Type hints for FormRequest --- scrapy/http/request/form.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ef2eb3ba6..4465f40ae 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,6 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode import lxml.html @@ -12,15 +13,18 @@ from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request +from scrapy.http.response.text import TextResponse from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url +FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") + + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, **kwargs): - formdata = kwargs.pop('formdata', None) + def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -36,9 +40,19 @@ class FormRequest(Request): self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) @classmethod - def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): - + def from_response( + cls: Type[FormRequestTypeVar], + response: TextResponse, + formname: Optional[str] = None, + formid: Optional[str] = None, + formnumber: Optional[int] = 0, + formdata: Optional[dict] = None, + clickdata: Optional[dict] = None, + dont_click: bool = False, + formxpath: Optional[str] = None, + formcss: Optional[str] = None, + **kwargs, + ) -> FormRequestTypeVar: kwargs.setdefault('encoding', response.encoding) if formcss is not None: From c594017e518af31edc24619bfc590907ba6280c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:27:21 -0300 Subject: [PATCH 1722/1853] Type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4465f40ae..781e3495a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode -import lxml.html +from lxml.html import HTMLParser, FormElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -72,7 +72,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form, url): +def _get_form_url(form: FormElement, url: Optional[str]) -> str: if url is None: action = form.get('action') if action is None: @@ -88,10 +88,15 @@ def _urlencode(seq, enc): return urlencode(values, doseq=True) -def _get_form(response, formname, formid, formnumber, formxpath): - """Find the form element """ - root = create_root_node(response.text, lxml.html.HTMLParser, - base_url=get_base_url(response)) +def _get_form( + response: TextResponse, + formname: Optional[str], + formid: Optional[str], + formnumber: Optional[int], + formxpath: Optional[str], +) -> FormElement: + """Find the wanted form element within the given response.""" + root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError(f"No
                element found in {response}") @@ -119,8 +124,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): break raise ValueError(f'No element found with {formxpath}') - # If we get here, it means that either formname was None - # or invalid + # If we get here, it means that either formname was None or invalid if formnumber is not None: try: form = forms[formnumber] From 85f88a5710e51a3137ded72f5fdb1c3465480355 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:02:16 -0300 Subject: [PATCH 1723/1853] More type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 781e3495a..1ad878c03 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Optional, Type, TypeVar +from typing import List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode from lxml.html import HTMLParser, FormElement @@ -20,11 +20,13 @@ from scrapy.utils.response import get_base_url FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: + def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -46,7 +48,7 @@ class FormRequest(Request): formname: Optional[str] = None, formid: Optional[str] = None, formnumber: Optional[int] = 0, - formdata: Optional[dict] = None, + formdata: FormdataType = None, clickdata: Optional[dict] = None, dont_click: bool = False, formxpath: Optional[str] = None, @@ -60,7 +62,7 @@ class FormRequest(Request): formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) - formdata = _get_inputs(form, formdata, dont_click, clickdata, response) + formdata = _get_inputs(form, formdata, dont_click, clickdata) url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) @@ -134,22 +136,27 @@ def _get_form( return form -def _get_inputs(form, formdata, dont_click, clickdata, response): +def _get_inputs( + form: FormElement, + formdata: FormdataType, + dont_click: bool, + clickdata: Optional[dict], +) -> List[Tuple[str, str]]: + """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') if not formdata: - formdata = () + formdata = [] inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' ' not(re:test(., "^(?:submit|image|reset)$", "i"))' ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', - namespaces={ - "re": "http://exslt.org/regular-expressions"}) + namespaces={"re": "http://exslt.org/regular-expressions"}) values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -160,7 +167,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.append(clickable) if isinstance(formdata, dict): - formdata = formdata.items() + formdata = formdata.items() # type: ignore[assignment] values.extend((k, v) for k, v in formdata if v is not None) return values @@ -189,7 +196,7 @@ def _select_value(ele, n, v): return n, v -def _get_clickable(clickdata, form): +def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first @@ -201,7 +208,7 @@ def _get_clickable(clickdata, form): namespaces={"re": "http://exslt.org/regular-expressions"} )) if not clickables: - return + return None # If we don't have clickdata, we just use the first clickable element if clickdata is None: From c9fecca010a3ddddd1145f65a78dde30b5c71a72 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:25:26 -0300 Subject: [PATCH 1724/1853] More type hints --- scrapy/http/request/form.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1ad878c03..c3e112041 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import List, Optional, Tuple, Type, TypeVar, Union +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode -from lxml.html import HTMLParser, FormElement +from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -83,7 +83,7 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: return urljoin(form.base_url, url) -def _urlencode(seq, enc): +def _urlencode(seq: Iterable, enc: str) -> str: values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] @@ -157,9 +157,11 @@ def _get_inputs( ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}) - values = [(k, '' if v is None else v) - for k, v in (_value(e) for e in inputs) - if k and k not in formdata_keys] + values = [ + (k, '' if v is None else v) + for k, v in (_value(e) for e in inputs) + if k and k not in formdata_keys + ] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -173,7 +175,7 @@ def _get_inputs( return values -def _value(ele): +def _value(ele: HtmlElement): n = ele.name v = ele.value if ele.tag == 'select': @@ -181,7 +183,7 @@ def _value(ele): return n, v -def _select_value(ele, n, v): +def _select_value(ele: SelectElement, n: str, v: str): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -192,7 +194,8 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or '').strip() for o in selected_options] + values = [(o.get('value') or o.text or '').strip() for o in selected_options] + return n, values return n, v From 479260dca012b3d03221f2e6322008448d0fb8b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:52:46 -0300 Subject: [PATCH 1725/1853] Type hints for Request subclasses --- scrapy/http/request/json_request.py | 17 +++++++---------- scrapy/http/request/rpc.py | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 04e80d897..dba3c3a82 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,9 +8,9 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Tuple +from typing import Optional, Tuple -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar from scrapy.utils.deprecate import create_deprecated_class @@ -18,8 +18,8 @@ class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, **kwargs): - dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs @@ -29,10 +29,8 @@ class JsonRequest(Request): if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -41,23 +39,22 @@ class JsonRequest(Request): self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @property - def dumps_kwargs(self): + def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs): + def replace(self, *args, **kwargs) -> RequestTypeVar: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) return super().replace(*args, **kwargs) - def _dumps(self, data): + def _dumps(self, data: dict) -> str: """Convert to JSON """ return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c70912e49..06d98cea5 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib +from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, **kwargs): - encoding = kwargs.get('encoding', None) + def __init__(self, *args, encoding: Optional[str] = None, **kwargs): if 'body' not in kwargs and 'params' in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs['body'] = xmlrpclib.dumps(**kw) From ce6447731a91e32faf17564d21ddc3b88e582f50 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 7 Jun 2021 13:25:04 -0300 Subject: [PATCH 1726/1853] Replace return type --- scrapy/http/request/__init__.py | 4 ++-- scrapy/http/request/json_request.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 8f00c20b7..7672dec00 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -126,10 +126,10 @@ class Request(object_ref): __repr__ = __str__ - def copy(self) -> RequestTypeVar: + def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index dba3c3a82..728a2a104 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ import json import warnings from typing import Optional, Tuple -from scrapy.http.request import Request, RequestTypeVar +from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class @@ -42,7 +42,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> Request: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None From e876d8e38780a88e4c1fc060e3ead779222be83a Mon Sep 17 00:00:00 2001 From: Veniamin Gvozdikov Date: Fri, 11 Jun 2021 09:22:04 +0300 Subject: [PATCH 1727/1853] Rename scrapy-crawlera to scrapy-zyte-smartproxy (#5074) --- scrapy/core/downloader/handlers/http11.py | 6 +++--- scrapy/core/downloader/handlers/http2.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 073f35891..50486d13c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -283,9 +283,9 @@ class ScrapyAgent: if omitConnectTunnel: warnings.warn( "Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Zyte Smart Proxy Manager (formerly Crawlera), " - "it doesn't require this mode anymore, so you should " - "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "If you use Zyte Smart Proxy Manager, it doesn't require " + "this mode anymore, so you should update scrapy-crawlera " + "to scrapy-zyte-smartproxy and remove '?noconnect' " "from the Zyte Smart Proxy Manager URL.", ScrapyDeprecationWarning, ) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e97c31e90..7bb88a193 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -72,10 +72,14 @@ class ScrapyH2Agent: proxy_host = proxy_host.decode() omit_connect_tunnel = b'noconnect' in proxy_params if omit_connect_tunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " - "downloader handler. If you use Crawlera, it doesn't require this " - "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.") + warnings.warn( + "Using HTTPS proxies in the noconnect mode is not " + "supported by the downloader handler. If you use Zyte " + "Smart Proxy Manager, it doesn't require this mode " + "anymore, so you should update scrapy-crawlera to " + "scrapy-zyte-smartproxy and remove '?noconnect' from the " + "Zyte Smart Proxy Manager URL." + ) if scheme == b'https' and not omit_connect_tunnel: # ToDo From 28858574d92fd2170575bea83fc6abac6b45b63a Mon Sep 17 00:00:00 2001 From: pdt1931 <56060869+pdt1931@users.noreply.github.com> Date: Fri, 11 Jun 2021 00:49:41 -0700 Subject: [PATCH 1728/1853] Add FAQ to code of Conduct (#5177) * added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct * added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct --- CODE_OF_CONDUCT.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 652460383..902cd523e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 66e200423943a921326334a37777720c706ee2b5 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 14 Jun 2021 18:58:35 +0500 Subject: [PATCH 1729/1853] Fix a flake8 problem --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ea32c6711..b610087bd 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -381,7 +381,8 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', + method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) From 5044549c550876cc31fb2e15aaa5013e8fc333c3 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Mon, 14 Jun 2021 14:58:19 +0000 Subject: [PATCH 1730/1853] Update proxyScheme assignment --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 15de8cdbd..d2f9084fc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,7 +319,7 @@ class ScrapyAgent: pool=self._pool, ) else: - proxyScheme = b'http' if not proxyScheme else proxyScheme + proxyScheme = proxyScheme or b'http' proxyHost = to_bytes(proxyHost, encoding='ascii') proxyPort = to_bytes(str(proxyPort), encoding='ascii') proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) From 7d653288e3d9369e4c007d223327f919b99b7737 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Mon, 14 Jun 2021 21:39:18 +0530 Subject: [PATCH 1731/1853] Update unittest --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 67224ed53..9c11820e5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -768,7 +768,7 @@ class Http11ProxyTestCase(HttpProxyTestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('').replace('http://', '') request = Request('http://example.com', meta={'proxy': http_proxy}) From 812b4bb51855605a36bcfb166f66a121cc67f97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Wed, 23 Jun 2021 17:09:28 +0100 Subject: [PATCH 1732/1853] CloseSpider can be raised on spider_idle signal handler --- docs/topics/signals.rst | 3 +++ scrapy/core/engine.py | 16 +++++++++++----- scrapy/utils/signal.py | 10 ++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3d838fb63..ee9d95836 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -268,6 +268,9 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. + Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` + exception to provide a custom spider closing reason. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd3225082..949d89e76 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,7 +15,8 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \ + CloseSpider from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider @@ -325,14 +326,19 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason """ assert self.spider is not None # typing - res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider) - if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): + expected_ex = (DontCloseSpider, CloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) + detected_ex = {ex: x.value + for _, x in res for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex)} + if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - self.close_spider(self.spider, reason='finished') + reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished' + self.close_spider(self.spider, reason=reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 115707182..62808f3ce 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" - +import collections import logging from twisted.internet.defer import DeferredList, Deferred @@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class _IgnoredException(Exception): - pass - - def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = (named.pop('dont_log', _IgnoredException), StopDownload) + dont_log = named.pop('dont_log', ()) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,) + dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): From ce445f20462463e49bd9dfd417aa7397d4c17f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Thu, 24 Jun 2021 09:56:05 +0100 Subject: [PATCH 1733/1853] Fix typing --- scrapy/core/engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 949d89e76..81a7a5068 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -337,8 +337,9 @@ class ExecutionEngine: if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished' - self.close_spider(self.spider, reason=reason) + ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished')) + assert isinstance(ex, CloseSpider) # typing + self.close_spider(self.spider, reason=ex.reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" From 73ff9ffd64742bb4d64db376851b52cfc3658fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 26 Jun 2021 08:58:29 +0200 Subject: [PATCH 1734/1853] spiders.rst: indent warnings into class descriptions --- docs/topics/spiders.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a3e9f410f..903fbd383 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -422,10 +422,9 @@ Crawling rules It receives a :class:`Twisted Failure ` instance as first parameter. - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`CrawlSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`CrawlSpider`-based spiders; + unexpected behaviour can occur otherwise. .. versionadded:: 2.0 The *errback* parameter. @@ -557,10 +556,9 @@ XMLFeedSpider item IDs. It receives a list of results and the response which originated those results. It must return a list of results (items or requests). - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; + unexpected behaviour can occur otherwise. XMLFeedSpider example From f35970778b93033ee36379e0b8b55035aacaf918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:21:38 +0100 Subject: [PATCH 1735/1853] Test case for raising CloseSpider on spider idle signal handler --- tests/test_engine.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c200ded90..dc24f50fa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -113,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider): } +class ChangeCloseReasonSpider(TestSpider): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + crawler.signals.connect(spider.spider_idle, signals.spider_idle) + return spider + + def spider_idle(self): + raise CloseSpider(reason="custom_reason") + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -251,6 +263,13 @@ class EngineTest(unittest.TestCase): yield self.run.run() self._assert_items_error() + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + self.run = CrawlerRun(ChangeCloseReasonSpider) + yield self.run.run() + self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, + self.run.signals_caught[signals.spider_closed]) + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] From e94d3ac173d2fcd269bdd8f06b6b1633953a225e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:40:43 +0100 Subject: [PATCH 1736/1853] Expanded doc for idle signal --- docs/topics/signals.rst | 6 +++++- scrapy/core/engine.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ee9d95836..530af1e37 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -269,7 +269,11 @@ spider_idle prevent the spider from being closed. Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` - exception to provide a custom spider closing reason. + exception to provide a custom spider closing reason. An + idle handler is the perfect place to put some code that assesses + the final spider results and update the final closing reason + accordingly (e.g. setting it to 'too_few_results' instead of + 'finished'). This signal does not support returning deferreds from its handlers. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 81a7a5068..0b34d213d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -326,7 +326,7 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. A handler can raise CloseSpider to provide a custom closing reason + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason. """ assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) From 7597d860c86e0ddfaf8e3ce57492106c907245f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:39:17 +0100 Subject: [PATCH 1737/1853] Update scrapy/core/engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0b34d213d..b7f91d744 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,8 +15,11 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \ - CloseSpider +from scrapy.exceptions import ( + CloseSpider, + DontCloseSpider, + ScrapyDeprecationWarning, +) from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider From 6b8f694653b0eec95055d50fafde0ef2e0ec8bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:40:02 +0100 Subject: [PATCH 1738/1853] Update scrapy/core/engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/engine.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7f91d744..b04fedce6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -334,9 +334,12 @@ class ExecutionEngine: assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) - detected_ex = {ex: x.value - for _, x in res for ex in expected_ex - if isinstance(x, Failure) and isinstance(x.value, ex)} + detected_ex = { + ex: x.value + for _, x in res + for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex) + } if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): From eca641aa3da335781c373e3d50deb78837044c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:40:20 +0100 Subject: [PATCH 1739/1853] Update tests/test_engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- tests/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index dc24f50fa..92bf45f25 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor From cb08e3644b081febecd90eec9391367a2d13ee5b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 8 Jul 2021 09:22:21 -0300 Subject: [PATCH 1740/1853] Remove trailing whitespaces --- scrapy/core/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b04fedce6..f9de7ee23 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,9 +17,9 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import ( CloseSpider, - DontCloseSpider, + DontCloseSpider, ScrapyDeprecationWarning, -) +) from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider From c062ed017a89e40c41140ff2782043776b26165b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 12 Jul 2021 13:34:22 -0300 Subject: [PATCH 1741/1853] [CI] fail-fast: false (#5200) --- .github/workflows/checks.yml | 1 + .github/workflows/tests-macos.yml | 1 + .github/workflows/tests-ubuntu.yml | 1 + .github/workflows/tests-windows.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 02c647da9..e7080db9a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -5,6 +5,7 @@ jobs: checks: runs-on: ubuntu-18.04 strategy: + fail-fast: false matrix: include: - python-version: 3.9 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 4f8f7a19d..095ca1013 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: macos-10.15 strategy: + fail-fast: false matrix: python-version: [3.6, 3.7, 3.8, 3.9] diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index df5ee9d69..b42e8b127 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: ubuntu-18.04 strategy: + fail-fast: false matrix: include: - python-version: 3.7 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5459a845b..30fda33e8 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: windows-latest strategy: + fail-fast: false matrix: include: - python-version: 3.6 From 4ddc9d6b55fa708becfd70812ea44eb0c6837638 Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Tue, 13 Jul 2021 20:52:29 +0530 Subject: [PATCH 1742/1853] Feeds: Item Filters (#5178) --- docs/topics/feed-exports.rst | 54 +++++++++++++++ scrapy/extensions/feedexport.py | 48 ++++++++++++- tests/test_feedexport.py | 118 ++++++++++++++++++++++++++++++-- 3 files changed, 214 insertions(+), 6 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 26c247cdd..7a4b054e9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. +.. _item-filter: + +Item filtering +============== + +.. versionadded:: VERSION + +You can filter items that you want to allow for a particular feed by using the +``item_classes`` option in :ref:`feeds options `. Only items of +the specified types will be added to the feed. + +The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter` +class, which is the default value of the ``item_filter`` :ref:`feed option `. + +You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s +method ``accepts`` and taking ``feed_options`` as an argument. + +For instance:: + + class MyCustomFilter: + + def __init__(self, feed_options): + self.feed_options = feed_options + + def accepts(self, item): + if "field1" in item and item["field1"] == "expected_data": + return True + return False + + +You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed `. +See :setting:`FEEDS` for examples. + +ItemFilter +---------- + +.. autoclass:: scrapy.extensions.feedexport.ItemFilter + :members: + Settings ======== @@ -311,6 +350,7 @@ For instance:: 'format': 'json', 'encoding': 'utf8', 'store_empty': False, + 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'], 'fields': None, 'indent': 4, 'item_export_kwargs': { @@ -320,12 +360,14 @@ For instance:: '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], + 'item_filter': MyCustomFilter1, 'encoding': 'latin1', 'indent': 8, }, pathlib.Path('items.csv'): { 'format': 'csv', 'fields': ['price', 'name'], + 'item_filter': 'myproject.filters.MyCustomFilter2', }, } @@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition: - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. +- ``item_classes``: list of :ref:`item classes ` to export. + + If undefined or empty, all items are exported. + + .. versionadded:: VERSION + +- ``item_filter``: a :ref:`filter class ` to filter items to export. + + :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. + + .. versionadded:: VERSION + - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bec114707..89dca12f4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): return builder(*preargs, uri, *args, **kwargs) +class ItemFilter: + """ + This will be used by FeedExporter to decide if an item should be allowed + to be exported to a particular feed. + + :param feed_options: feed specific options passed from FeedExporter + :type feed_options: dict + """ + + def __init__(self, feed_options): + self.feed_options = feed_options + self.item_classes = set() + + if 'item_classes' in self.feed_options: + for item_class in self.feed_options['item_classes']: + self.item_classes.add(load_object(item_class)) + + def accepts(self, item): + """ + Return ``True`` if `item` should be exported or ``False`` otherwise. + + :param item: scraped item which user wants to check if is acceptable + :type item: :ref:`Scrapy items ` + :return: `True` if accepted, `False` otherwise + :rtype: bool + """ + if self.item_classes: + return isinstance(item, tuple(self.item_classes)) + + return True # accept all items if none declared in item_classes + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" @@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter): self.file = file self.exporter = exporter self.storage = storage @@ -225,6 +257,7 @@ class _FeedSlot: self.store_empty = store_empty self.uri_template = uri_template self.uri = uri + self.filter = filter # flags self.itemcount = 0 self._exporting = False @@ -255,6 +288,7 @@ class FeedExporter: self.settings = crawler.settings self.feeds = {} self.slots = [] + self.filters = {} if not self.settings['FEEDS'] and not self.settings['FEED_URI']: raise NotConfigured @@ -269,12 +303,14 @@ class FeedExporter: uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') @@ -368,6 +404,7 @@ class FeedExporter: store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, + filter=self.filters[uri_template] ) if slot.store_empty: slot.start_exporting() @@ -376,6 +413,10 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] for slot in self.slots: + if not slot.filter.accepts(item): + slots.append(slot) # if slot doesn't accept item, continue with next slot + continue + slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 @@ -486,3 +527,8 @@ class FeedExporter: uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params + + def _load_filter(self, feed_options): + # load the item filter if declared else load the default filter class + item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + return item_filter_class(feed_options) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index df7ec4461..81437b011 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): egg = scrapy.Field() baz = scrapy.Field() + class MyItem2(scrapy.Item): + foo = scrapy.Field() + hello = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) @@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - MyItem2({'hello': 'world2', 'foo': 'bar2'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}), {'hello': 'world4', 'egg': 'spam4'}, ] @@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + @defer.inlineCallbacks + def test_export_based_on_item_classes(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + formats = { + 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n', + 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]', + 'jsonlines': ( + b'{"foo": "bar1", "egg": "spam1"}\n' + b'{"hello": "world2", "foo": "bar2"}\n' + ), + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\nworld3' + b'spam3\n' + ), + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'csv', + 'item_classes': [self.MyItem], + }, + self._random_temp_filename(): { + 'format': 'json', + 'item_classes': [self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'xml', + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + + @defer.inlineCallbacks + def test_export_based_on_custom_filters(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + MyItem = self.MyItem + + class CustomFilter1: + def __init__(self, feed_options): + pass + + def accepts(self, item): + return isinstance(item, MyItem) + + class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if 'foo' not in item.fields: + return False + return True + + class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1": + return True + return False + + formats = { + 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]', + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\n' + ), + 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n', + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'json', + 'item_filter': CustomFilter1, + }, + self._random_temp_filename(): { + 'format': 'xml', + 'item_filter': CustomFilter2, + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + 'item_filter': CustomFilter3, + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as From fcc6becc586e6f895cdeed66a579d5735d215de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=BCrkalp=20Burak=20KAYRANCIO=C4=9ELU?= Date: Wed, 14 Jul 2021 11:00:43 +0300 Subject: [PATCH 1743/1853] S3FeedStorage: allow custom endpoint (#4998) Co-authored-by: Andrey Rahmatullin --- docs/topics/feed-exports.rst | 3 ++- scrapy/extensions/feedexport.py | 7 ++++-- tests/test_feedexport.py | 44 +++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7a4b054e9..af7fce852 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -201,9 +201,10 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` -You can also define a custom ACL for exported feeds using this setting: +You can also define a custom ACL and custom endpoint for exported feeds using this setting: - :setting:`FEED_STORAGE_S3_ACL` +- :setting:`AWS_ENDPOINT_URL` This storage backend uses :ref:`delayed file delivery `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 89dca12f4..84a79e32d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -150,7 +150,7 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, feed_options=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.keyname = u.path[1:] # remove first "/" self.acl = acl + self.endpoint_url = endpoint_url import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) + aws_secret_access_key=self.secret_key, + endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage): access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 81437b011..da0b2c786 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + endpoint_url='https://example.com' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + def test_from_crawler_without_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def test_without_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, None) + def test_from_crawler_with_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_from_crawler_with_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'AWS_ENDPOINT_URL': 'https://example.com', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): skip_if_no_boto() @@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl): - super().__init__(uri, access_key, secret_key, acl) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url): + super().__init__(uri, access_key, secret_key, acl, endpoint_url) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): From d7deba7e89242774ae712d87eb7bb331759a731f Mon Sep 17 00:00:00 2001 From: Marlena Chatzigrigoriou <56519084+marlenachatzigrigoriou@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:34:28 +0300 Subject: [PATCH 1744/1853] Document all import paths and use the shortest in examples (#5099) --- docs/faq.rst | 4 +- docs/intro/tutorial.rst | 32 +++++++-------- docs/topics/api.rst | 4 +- docs/topics/contracts.rst | 4 +- docs/topics/coroutines.rst | 2 +- docs/topics/debug.rst | 2 +- docs/topics/developer-tools.rst | 2 +- docs/topics/downloader-middleware.rst | 58 +++++++++++++-------------- docs/topics/dynamic-content.rst | 14 +++---- docs/topics/exporters.rst | 4 +- docs/topics/feed-exports.rst | 4 +- docs/topics/item-pipeline.rst | 6 +-- docs/topics/items.rst | 10 +++-- docs/topics/jobs.rst | 4 +- docs/topics/leaks.rst | 16 ++++---- docs/topics/logging.rst | 2 +- docs/topics/media-pipeline.rst | 6 +-- docs/topics/request-response.rst | 16 ++++---- docs/topics/selectors.rst | 8 ++-- docs/topics/settings.rst | 16 ++++---- docs/topics/shell.rst | 8 ++-- docs/topics/signals.rst | 54 ++++++++++++------------- docs/topics/spider-middleware.rst | 26 ++++++------ docs/topics/spiders.rst | 43 ++++++++++---------- 24 files changed, 175 insertions(+), 170 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9709885f6..16903daea 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ If you have a spider with a long list of -:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider replacing the default :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware with a :ref:`custom spider middleware ` that requires @@ -157,7 +157,7 @@ less memory. For example: - If your domain names are similar enough, use your own regular expression instead joining the strings in - :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. - If you can `meet the installation requirements`_, use pyre2_ instead of diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 740e47d0c..438f3d6df 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -78,7 +78,7 @@ Our first Spider Spiders are classes that you define and that Scrapy uses to scrape information from a website (or a group of websites). They must subclass -:class:`~scrapy.spiders.Spider` and define the initial requests to make, +:class:`~scrapy.Spider` and define the initial requests to make, optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. @@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named self.log(f'Saved file {filename}') -As you can see, our Spider subclasses :class:`scrapy.Spider ` +As you can see, our Spider subclasses :class:`scrapy.Spider ` and defines some attributes and methods: -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be +* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of +* :meth:`~scrapy.Spider.start_requests`: must return an iterable of Requests (you can return a list of requests or write a generator function) which the Spider will begin to crawl from. Subsequent requests will be generated successively from these initial requests. -* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle +* :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter is an instance of :class:`~scrapy.http.TextResponse` that holds the page content and has further helpful methods to handle it. - The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting the scraped data as dicts and also finding new URLs to - follow and creating new requests (:class:`~scrapy.http.Request`) from them. + follow and creating new requests (:class:`~scrapy.Request`) from them. How to run our spider --------------------- @@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects +Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the @@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the A shortcut to the start_requests method --------------------------------------- -Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +Instead of implementing a :meth:`~scrapy.Spider.start_requests` method +that generates :class:`scrapy.Request ` objects from URLs, +you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: import scrapy @@ -194,9 +194,9 @@ for your spider:: with open(filename, 'wb') as f: f.write(response.body) -The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy -to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's default callback method, which is called for requests without an explicitly assigned callback. @@ -248,7 +248,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of -:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +:class:`~scrapy.Selector` objects that wrap around XML/HTML elements and allow you to run further queries to fine-grain the selection or extract the data. @@ -670,7 +670,7 @@ the pagination links with the ``parse`` callback as we saw before. Here we're passing callbacks to :meth:`response.follow_all ` as positional arguments to make the code shorter; it also works for -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 445b2979f..900b19c7a 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -29,7 +29,7 @@ how you :ref:`configure the downloader middlewares .. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a - :class:`scrapy.spiders.Spider` subclass and a + :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -196,7 +196,7 @@ SpiderLoader API match the request's url against the domains of the spiders. :param request: queried request - :type request: :class:`~scrapy.http.Request` instance + :type request: :class:`~scrapy.Request` instance .. _topics-api-signals: diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index e61421bf1..ef296dc9e 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -37,7 +37,7 @@ This callback is tested using three built-in contracts: .. class:: CallbackKeywordArgumentsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` + This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` attribute for the sample request. It must be a valid JSON dictionary. :: @@ -88,7 +88,7 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used by default, + for request object. :class:`~scrapy.Request` is used by default, but this can be changed with the ``request_cls`` attribute. If multiple contracts in chain have this attribute defined, the last one is used. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3b1549bd3..0904637b0 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -15,7 +15,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- :class:`~scrapy.http.Request` callbacks. +- :class:`~scrapy.Request` callbacks. .. note:: The callback output is not processed until the whole callback finishes. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d75f17301..4d452b4df 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -36,7 +36,7 @@ Consider the following Scrapy spider below:: Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a partially populated item. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c83b1a9d9..057b1ec62 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -274,7 +274,7 @@ In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl()` method to generate an equivalent request:: from scrapy import Request diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b539c23df..80c6c2c37 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings `. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -88,7 +88,7 @@ object gives you access, for example, to the :ref:`settings `. or the appropriate download function; it'll return that response. The :meth:`process_response` methods of installed middleware is always called on every response. - If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling + If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling process_request methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. @@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings `. ignored and not logged (unlike other exceptions). :param request: the request being processed - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_response(request, response, spider) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` - object, return a :class:`~scrapy.http.Request` object or + object, return a :class:`~scrapy.Request` object or raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception. If it returns a :class:`~scrapy.http.Response` (it could be the same given response, or a brand-new one), that response will continue to be processed with the :meth:`process_response` of the next middleware in the chain. - If it returns a :class:`~scrapy.http.Request` object, the middleware chain is + If it returns a :class:`~scrapy.Request` object, the middleware chain is halted and the returned request is rescheduled to be downloaded in the future. This is the same behavior as if a request is returned from :meth:`process_request`. @@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings `. exception, it is ignored and not logged (unlike other exceptions). :param request: the request that originated the response - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param response: the response being processed :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_exception(request, exception, spider) @@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings `. exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) :meth:`process_exception` should return: either ``None``, - a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object. + a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_exception` methods of installed middleware, @@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings `. method chain of installed middleware is started, and Scrapy won't bother calling any other :meth:`process_exception` methods of middleware. - If it returns a :class:`~scrapy.http.Request` object, the returned request is + If it returns a :class:`~scrapy.Request` object, the returned request is rescheduled to be downloaded in the future. This stops the execution of :meth:`process_exception` methods of the middleware the same as returning a response would. :param request: the request that generated the exception - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param exception: the raised exception :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -203,13 +203,13 @@ CookiesMiddleware browsers do. .. caution:: When non-UTF8 encoded byte sequences are passed to a - :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. The following settings can be used to configure the cookie middleware: @@ -258,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will **not** be merged with the existing cookies. For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. .. setting:: COOKIES_DEBUG @@ -501,7 +501,7 @@ defines the methods described below. the :signal:`open_spider ` signal. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(spider) @@ -509,27 +509,27 @@ defines the methods described below. the :signal:`close_spider ` signal. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: retrieve_response(spider, request) Return response if present in cache, or ``None`` otherwise. :param spider: the spider which generated the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the request to find cached response for - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object .. method:: store_response(spider, request, response) Store the given response in the cache. :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the corresponding request the spider generated - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param response: the response to store in the cache :type response: :class:`~scrapy.http.Response` object @@ -722,7 +722,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.http.Request` objects. + ``proxy`` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -749,12 +749,12 @@ RedirectMiddleware .. reqmeta:: redirect_urls The urls which the request goes through (while being redirected) can be found -in the ``redirect_urls`` :attr:`Request.meta ` key. +in the ``redirect_urls`` :attr:`Request.meta ` key. .. reqmeta:: redirect_reasons The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the -``redirect_reasons`` :attr:`Request.meta ` key. For +``redirect_reasons`` :attr:`Request.meta ` key. For example: ``[301, 302, 307, 'meta refresh']``. The format of a reason depends on the middleware that handled the corresponding @@ -770,7 +770,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_redirect -If :attr:`Request.meta ` has ``dont_redirect`` +If :attr:`Request.meta ` has ``dont_redirect`` key set to True, the request will be ignored by this middleware. If you want to handle some redirect status codes in your spider, you can @@ -783,7 +783,7 @@ responses (and pass them through to your spider) you can do this:: handle_httpstatus_list = [301, 302] The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request. @@ -889,7 +889,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_retry -If :attr:`Request.meta ` has ``dont_retry`` key +If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. To retry requests from a spider callback, you can use the @@ -919,7 +919,7 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. Maximum number of retries can also be specified per-request using -:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. @@ -986,7 +986,7 @@ RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt -If :attr:`Request.meta ` has +If :attr:`Request.meta ` has ``dont_obey_robotstxt`` key set to True the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 495111b56..aa32c8943 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the information can be found in the response they get. If they get a response with the desired data, modify your Scrapy -:class:`~scrapy.http.Request` to match that of the other HTTP client. For +:class:`~scrapy.Request` to match that of the other HTTP client. For example, try using the same user-agent string (:setting:`USER_AGENT`) or the -same :attr:`~scrapy.http.Request.headers`. +same :attr:`~scrapy.Request.headers`. If they also get a response without the desired data, you’ll need to take steps to make your request more similar to that of the web browser. See @@ -81,14 +81,14 @@ Use the :ref:`network tool ` of your web browser to see how your web browser performs the desired request, and try to reproduce that request with Scrapy. -It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP +It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.http.FormRequest`) of that request. +form parameters (see :class:`~scrapy.FormRequest`) of that request. As all major browsers allow to export the requests in `cURL `_ format, Scrapy incorporates the method -:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent -:class:`~scrapy.http.Request` from a cURL command. To get more information +:meth:`~scrapy.Request.from_curl()` to generate an equivalent +:class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -125,7 +125,7 @@ data from it depends on the type of response: If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a - :class:`~scrapy.selector.Selector` and then + :class:`~scrapy.Selector` and then :ref:`use it ` as usual:: selector = Selector(data['html']) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8648daded..8c30122b6 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -90,7 +90,7 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~.Item` you can declare a serializer in the +If you use :class:`~scrapy.Item` you can declare a serializer in the :ref:`field metadata `. The serializer must be a callable which receives a value and returns its serialized form. @@ -172,7 +172,7 @@ BaseItemExporter :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty :class:`dict`. - :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance + :type field: :class:`~scrapy.Field` object or a :class:`dict` instance :param name: the name of the field being serialized :type name: str diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index af7fce852..216a8bc52 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -674,9 +674,9 @@ The function signature should be as follows: :type params: dict :param spider: source spider of the feed items - :type spider: scrapy.spiders.Spider + :type spider: scrapy.Spider -For example, to include the :attr:`name ` of the +For example, to include the :attr:`name ` of the source spider in the feed URI: #. Define the following function somewhere in your project:: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 6287ee0ad..5351a2293 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Additionally, they may also implement the following methods: @@ -51,14 +51,14 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. :param spider: the spider which was opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(self, spider) This method is called when the spider is closed. :param spider: the spider which was closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 65bf156ac..7cd482d07 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -42,7 +42,8 @@ Item objects :class:`Item` provides a :class:`dict`-like API plus additional features that make it the most feature-complete item type: -.. class:: Item([arg]) +.. class:: scrapy.item.Item([arg]) +.. class:: scrapy.Item([arg]) :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. @@ -199,7 +200,8 @@ It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through the :attr:`Item.fields` attribute. -.. class:: Field([arg]) +.. class:: scrapy.item.Field([arg]) +.. class:: scrapy.Field([arg]) The :class:`Field` class is just an alias to the built-in :class:`dict` class and doesn't provide any extra functionality or attributes. In other words, @@ -317,11 +319,11 @@ If that is not the desired behavior, use a deep copy instead. See :mod:`copy` for more information. To create a shallow copy of an item, you can either call -:meth:`~scrapy.item.Item.copy` on an existing item +:meth:`~scrapy.Item.copy` on an existing item (``product2 = product.copy()``) or instantiate your item class from an existing item (``product2 = Product(product)``). -To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead (``product2 = product.deepcopy()``). diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index d855d0133..e49f37a2f 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -74,10 +74,10 @@ on cookies. Request serialization --------------------- -For persistence to work, :class:`~scrapy.http.Request` objects must be +For persistence to work, :class:`~scrapy.Request` objects must be serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.spiders.Spider` class. +running :class:`~scrapy.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index b895b95cb..477652704 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -27,7 +27,7 @@ Common causes of memory leaks It happens quite often (sometimes by accident, sometimes on purpose) that the Scrapy developer passes objects referenced in Requests (for example, using the -:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta` +:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta` attributes or the request callback function) and that effectively bounds the lifetime of those referenced objects to the lifetime of the Request. This is, by far, the most common cause of memory leaks in Scrapy projects, and a quite @@ -48,9 +48,9 @@ Too Many Requests? ------------------ By default Scrapy keeps the request queue in memory; it includes -:class:`~scrapy.http.Request` objects and all objects -referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs` -and :attr:`~scrapy.http.Request.meta`). +:class:`~scrapy.Request` objects and all objects +referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs` +and :attr:`~scrapy.Request.meta`). While not necessarily a leak, this can take a lot of memory. Enabling :ref:`persistent job queue ` could help keeping memory usage in control. @@ -90,11 +90,11 @@ Which objects are tracked? The objects tracked by ``trackrefs`` are all from these classes (and all its subclasses): -* :class:`scrapy.http.Request` +* :class:`scrapy.Request` * :class:`scrapy.http.Response` -* :class:`scrapy.item.Item` -* :class:`scrapy.selector.Selector` -* :class:`scrapy.spiders.Spider` +* :class:`scrapy.Item` +* :class:`scrapy.Selector` +* :class:`scrapy.Spider` A real example -------------- diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 00806392a..dda04dc4d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -93,7 +93,7 @@ path:: Logging from Spiders ==================== -Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider +Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider instance, which can be accessed and used like this:: import scrapy diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 156897274..3438cb637 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -259,7 +259,7 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.item.Item` class:: +``images`` field. For instance, using the :class:`~scrapy.Item` class:: import scrapy @@ -424,7 +424,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. @@ -563,7 +563,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 73b5a858f..a6a3daf31 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -35,7 +35,7 @@ Request objects request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.spiders.Spider.parse` method will be used. + :meth:`~scrapy.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. :type callback: collections.abc.Callable @@ -60,7 +60,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type headers: dict @@ -92,7 +92,7 @@ Request objects To create a request that does not send stored cookies and does not store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. + in :attr:`request.meta `. Example of a request that sends manually-defined cookies and ignores cookie storage:: @@ -107,7 +107,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type cookies: dict or list @@ -495,7 +495,9 @@ fields with form data from :class:`Response` objects. .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms -.. class:: FormRequest(url, [formdata, ...]) +.. class:: scrapy.http.request.form.FormRequest +.. class:: scrapy.http.FormRequest +.. class:: scrapy.FormRequest(url, [formdata, ...]) The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The remaining arguments are the same as for the :class:`Request` class and are @@ -694,7 +696,7 @@ Response objects :param request: the initial value of the :attr:`Response.request` attribute. This represents the :class:`Request` that generated this response. - :type request: scrapy.http.Request + :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. :type certificate: twisted.internet.ssl.Certificate @@ -920,7 +922,7 @@ TextResponse objects .. attribute:: TextResponse.selector - A :class:`~scrapy.selector.Selector` instance using the response as + A :class:`~scrapy.Selector` instance using the response as target. The selector is lazily instantiated on first access. :class:`TextResponse` objects support the following methods in addition to diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9caba5ee5..574d4568c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -48,7 +48,7 @@ Constructing selectors .. highlight:: python -Response objects expose a :class:`~scrapy.selector.Selector` instance +Response objects expose a :class:`~scrapy.Selector` instance on ``.selector`` attribute: >>> response.selector.xpath('//span/text()').get() @@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: >>> response.css('span::text').get() 'good' -Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +Scrapy selectors are instances of :class:`~scrapy.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or markup as a string (in ``text`` argument). @@ -175,7 +175,7 @@ of ``None``: 'not-found' Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes -using ``.attrib`` property of a :class:`~scrapy.selector.Selector`: +using ``.attrib`` property of a :class:`~scrapy.Selector`: >>> [img.attrib['src'] for img in response.css('img')] ['image1_thumb.jpg', @@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page): Using selectors with regular expressions ---------------------------------------- -:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting +:class:`~scrapy.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or ``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b290598f..1290b4a5e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -67,7 +67,7 @@ Example:: Spiders (See the :ref:`topics-spiders` chapter for reference) can define their own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute:: +do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:: class MySpider(scrapy.Spider): name = 'myspider' @@ -142,7 +142,7 @@ In a spider, the settings are available through ``self.settings``:: The ``settings`` attribute is set in the base Spider class after the spider is initialized. If you want to use the settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the - :meth:`~scrapy.spiders.Spider.from_crawler` method. + :meth:`~scrapy.Spider.from_crawler` method. Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in @@ -338,7 +338,7 @@ is non-zero, download delay is enforced per IP, not per domain. DEFAULT_ITEM_CLASS ------------------ -Default: ``'scrapy.item.Item'`` +Default: ``'scrapy.Item'`` The default class that will be used for instantiating items in the :ref:`the Scrapy shell `. @@ -360,7 +360,7 @@ The default headers used for Scrapy HTTP Requests. They're populated in the .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. .. setting:: DEPTH_LIMIT @@ -384,8 +384,8 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of -a :class:`~scrapy.http.Request` based on its depth. +An integer that is used to adjust the :attr:`~scrapy.Request.priority` of +a :class:`~scrapy.Request` based on its depth. The priority of a request is adjusted as follows:: @@ -816,14 +816,14 @@ The default (``RFPDupeFilter``) filters based on request fingerprint using the ``scrapy.utils.request.request_fingerprint`` function. In order to change the way duplicates are checked you could subclass ``RFPDupeFilter`` and override its ``request_fingerprint`` method. This method should accept -scrapy :class:`~scrapy.http.Request` object and return its fingerprint +scrapy :class:`~scrapy.Request` object and return its fingerprint (a string). You can disable filtering of duplicate requests by setting :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. Be very careful about this however, because you can get into crawling loops. It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.http.Request` that should not be +``True`` on the specific :class:`~scrapy.Request` that should not be filtered. .. setting:: DUPEFILTER_DEBUG diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index b910fc453..8c90a506c 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -118,7 +118,7 @@ Available Scrapy objects The Scrapy shell automatically creates some convenient objects from the downloaded page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.Selector` objects (for both HTML and XML +:class:`~scrapy.Selector` objects (for both HTML and XML content). Those objects are: @@ -126,12 +126,12 @@ Those objects are: - ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for the + :class:`~scrapy.Spider` object if there is no spider found for the current URL -- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched +- ``request`` - a :class:`~scrapy.Request` object of the last fetched page. You can modify this request using - :meth:`~scrapy.http.Request.replace` or fetch a new request (without + :meth:`~scrapy.Request.replace` or fetch a new request (without leaving the shell) using the ``fetch`` shortcut. - ``response`` - a :class:`~scrapy.http.Response` object containing the last diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 530af1e37..a67cc1879 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -155,7 +155,7 @@ item_scraped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped :type response: :class:`~scrapy.http.Response` object @@ -175,7 +175,7 @@ item_dropped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped :type response: :class:`~scrapy.http.Response` object @@ -203,7 +203,7 @@ item_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -223,7 +223,7 @@ spider_closed This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param reason: a string which describes the reason why the spider was closed. If it was closed because the spider has completed scraping, the reason @@ -247,7 +247,7 @@ spider_opened This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object spider_idle ~~~~~~~~~~~ @@ -278,7 +278,7 @@ spider_idle This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: Scheduling some requests in your :signal:`spider_idle` handler does **not** guarantee that it can prevent the spider from being closed, @@ -303,7 +303,7 @@ spider_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Request signals --------------- @@ -314,16 +314,16 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.http.Request`, to be + Sent when the engine schedules a :class:`~scrapy.Request`, to be downloaded later. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_dropped ~~~~~~~~~~~~~~~ @@ -331,16 +331,16 @@ request_dropped .. signal:: request_dropped .. function:: request_dropped(request, spider) - Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be + Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_reached_downloader ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -348,15 +348,15 @@ request_reached_downloader .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` reached downloader. + Sent when a :class:`~scrapy.Request` reached downloader. This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_left_downloader ~~~~~~~~~~~~~~~~~~~~~~~ @@ -366,16 +366,16 @@ request_left_downloader .. versionadded:: 2.0 - Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of + Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. This signal does not support returning deferreds from its handlers. :param request: the request that reached the downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object bytes_received ~~~~~~~~~~~~~~ @@ -402,10 +402,10 @@ bytes_received :type data: :class:`bytes` object :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object headers_received ~~~~~~~~~~~~~~~~ @@ -432,10 +432,10 @@ headers_received :type body_length: `int` :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Response signals ---------------- @@ -455,10 +455,10 @@ response_received :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: The ``request`` argument might not contain the original request that reached the downloader, if a :ref:`topics-downloader-middleware` modifies @@ -479,7 +479,7 @@ response_downloaded :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 11bbbb58d..f0158dc41 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output(response, result, spider) @@ -102,7 +102,7 @@ object gives you access, for example, to the :ref:`settings `. it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request` objects and :ref:`item object + :class:`~scrapy.Request` objects and :ref:`item object `. :param response: the response which generated this output from the @@ -110,11 +110,11 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request` objects and + :type result: an iterable of :class:`~scrapy.Request` objects and :ref:`item object ` :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_exception(response, exception, spider) @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` objects and :ref:`item object + iterable of :class:`~scrapy.Request` objects and :ref:`item object `. If it returns ``None``, Scrapy will continue processing this exception, @@ -142,7 +142,7 @@ object gives you access, for example, to the :ref:`settings `. :type exception: :exc:`Exception` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_start_requests(start_requests, spider) @@ -152,7 +152,7 @@ object gives you access, for example, to the :ref:`settings `. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.http.Request` objects. + return another iterable of :class:`~scrapy.Request` objects. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and @@ -164,10 +164,10 @@ object gives you access, for example, to the :ref:`settings `. (like a time limit or item/page count). :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.http.Request` + :type start_requests: an iterable of :class:`~scrapy.Request` :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -251,7 +251,7 @@ this:: .. reqmeta:: handle_httpstatus_all The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request, and ``False`` to disable the effects of the ``handle_httpstatus_all`` key. @@ -295,7 +295,7 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. All subdomains of any domain in the list are also allowed. E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` but not ``www2.example.com`` nor ``example.com``. @@ -313,10 +313,10 @@ OffsiteMiddleware will be printed (but only for the first request filtered). If the spider doesn't define an - :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the + :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. - If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set, the offsite middleware will allow the request even if its domain is not listed in allowed domains. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 903fbd383..67b9e2e0e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by calling the - :meth:`~scrapy.spiders.Spider.start_requests` method which (by default) - generates :class:`~scrapy.http.Request` for the URLs specified in the - :attr:`~scrapy.spiders.Spider.start_urls` and the - :attr:`~scrapy.spiders.Spider.parse` method as callback function for the + :meth:`~scrapy.Spider.start_requests` method which (by default) + generates :class:`~scrapy.Request` for the URLs specified in the + :attr:`~scrapy.Spider.start_urls` and the + :attr:`~scrapy.Spider.parse` method as callback function for the Requests. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, - :class:`~scrapy.http.Request` objects, or an iterable of these objects. + :class:`~scrapy.Request` objects, or an iterable of these objects. Those Requests will also contain a callback (maybe the same) and will then be downloaded by Scrapy and then their response handled by the specified callback. @@ -50,7 +50,8 @@ We will talk about those types here. scrapy.Spider ============= -.. class:: Spider() +.. class:: scrapy.spiders.Spider() +.. class:: scrapy.Spider() This is the simplest spider, and the one from which every other spider must inherit (including spiders that come bundled with Scrapy, as well as spiders @@ -86,7 +87,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data + listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings @@ -179,7 +180,7 @@ scrapy.Spider the same requirements as the :class:`Spider` class. This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects + iterable of :class:`~scrapy.Request` and/or :ref:`item objects `. :param response: the response to parse @@ -234,7 +235,7 @@ Return multiple Requests and items from a single callback:: yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.item.Item` objects:: +to give data more structure you can use :class:`~scrapy.Item` objects:: import scrapy from myproject.items import MyItem @@ -373,7 +374,7 @@ CrawlSpider This method is called for each response produced for the URLs in the spider's ``start_urls`` attribute. It allows to parse the initial responses and must return either an - :ref:`item object `, a :class:`~scrapy.http.Request` + :ref:`item object `, a :class:`~scrapy.Request` object, or an iterable containing any of them. Crawling rules @@ -383,7 +384,7 @@ Crawling rules ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. Each produced link will - be used to generate a :class:`~scrapy.http.Request` object, which will contain the + be used to generate a :class:`~scrapy.Request` object, which will contain the link's text in its ``meta`` dictionary (under the ``link_text`` key). If omitted, a default link extractor created with no arguments will be used, resulting in all links being extracted. @@ -392,9 +393,9 @@ Crawling rules object with that name will be used) to be called for each link extracted with the specified link extractor. This callback receives a :class:`~scrapy.http.Response` as its first argument and must return either a single instance or an iterable of - :ref:`item objects ` and/or :class:`~scrapy.http.Request` objects + :ref:`item objects ` and/or :class:`~scrapy.Request` objects (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response` - object will contain the text of the link that produced the :class:`~scrapy.http.Request` + object will contain the text of the link that produced the :class:`~scrapy.Request` in its ``meta`` dictionary (under the ``link_text`` key) ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the @@ -411,7 +412,7 @@ Crawling rules ``process_request`` is a callable (or a string, in which case a method from the spider object with that name will be used) which will be called for every - :class:`~scrapy.http.Request` extracted by this rule. This callable should + :class:`~scrapy.Request` extracted by this rule. This callable should take said request as first argument and the :class:`~scrapy.http.Response` from which the request originated as second argument. It must return a ``Request`` object or ``None`` (to filter out the request). @@ -470,7 +471,7 @@ Let's now take a look at an example CrawlSpider with rules:: This spider would start crawling example.com's home page, collecting category links, and item links, parsing the latter with the ``parse_item`` method. For each item response, some data will be extracted from the HTML using XPath, and -an :class:`~scrapy.item.Item` will be filled with it. +an :class:`~scrapy.Item` will be filled with it. XMLFeedSpider ------------- @@ -493,11 +494,11 @@ XMLFeedSpider - ``'iternodes'`` - a fast iterator based on regular expressions - - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds - - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds @@ -515,7 +516,7 @@ XMLFeedSpider available in that document that will be processed with this spider. The ``prefix`` and ``uri`` will be used to automatically register namespaces using the - :meth:`~scrapy.selector.Selector.register_namespace` method. + :meth:`~scrapy.Selector.register_namespace` method. You can then specify nodes with namespaces in the :attr:`itertag` attribute. @@ -542,10 +543,10 @@ XMLFeedSpider This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an - :class:`~scrapy.selector.Selector` for each node. Overriding this + :class:`~scrapy.Selector` for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method must return an :ref:`item object `, a - :class:`~scrapy.http.Request` object, or an iterable containing any of + :class:`~scrapy.Request` object, or an iterable containing any of them. .. method:: process_results(response, results) @@ -587,7 +588,7 @@ These spiders are pretty easy to use, let's have a look at one example:: Basically what we did up there was to create a spider that downloads a feed from the given ``start_urls``, and then iterates through each of its ``item`` tags, -prints them out, and stores some random data in an :class:`~scrapy.item.Item`. +prints them out, and stores some random data in an :class:`~scrapy.Item`. CSVFeedSpider ------------- From bcce0660573cf9309e3decf853488da2ef8bc576 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 14 Jul 2021 12:56:07 -0300 Subject: [PATCH 1745/1853] Update ItemFilter (#5203) --- scrapy/extensions/feedexport.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 84a79e32d..bd4808e2b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,6 +11,7 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile +from typing import Any, Optional, Tuple from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -54,16 +55,19 @@ class ItemFilter: :param feed_options: feed specific options passed from FeedExporter :type feed_options: dict """ + feed_options: Optional[dict] + item_classes: Tuple - def __init__(self, feed_options): + def __init__(self, feed_options: Optional[dict]) -> None: self.feed_options = feed_options - self.item_classes = set() + if feed_options is not None: + self.item_classes = tuple( + load_object(item_class) for item_class in feed_options.get("item_classes") or () + ) + else: + self.item_classes = tuple() - if 'item_classes' in self.feed_options: - for item_class in self.feed_options['item_classes']: - self.item_classes.add(load_object(item_class)) - - def accepts(self, item): + def accepts(self, item: Any) -> bool: """ Return ``True`` if `item` should be exported or ``False`` otherwise. @@ -73,9 +77,8 @@ class ItemFilter: :rtype: bool """ if self.item_classes: - return isinstance(item, tuple(self.item_classes)) - - return True # accept all items if none declared in item_classes + return isinstance(item, self.item_classes) + return True # accept all items by default class IFeedStorage(Interface): From 89b654b82df763a59d6a37e92e796c1478c768ce Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 16 Jul 2021 15:18:14 +0500 Subject: [PATCH 1746/1853] Make the pylint test pass (#5207) Co-authored-by: Vostretsov Nikita --- .github/workflows/checks.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ pylintrc | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e7080db9a..6bdfcb5dc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,6 +19,7 @@ jobs: - python-version: 3.8 env: TOXENV: pylint + TOX_PIP_VERSION: 20.3.3 - python-version: 3.9 env: TOXENV: typing @@ -37,5 +38,8 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi pip install -U tox tox diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index b42e8b127..521d7ae70 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -40,6 +40,7 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps + TOX_PIP_VERSION: 20.3.3 - python-version: 3.9 env: TOXENV: asyncio @@ -68,6 +69,9 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi pip install -U tox tox diff --git a/pylintrc b/pylintrc index 972bf99de..a44712507 100644 --- a/pylintrc +++ b/pylintrc @@ -6,6 +6,7 @@ jobs=1 # >1 hides results disable=abstract-method, anomalous-backslash-in-string, arguments-differ, + arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, bad-continuation, @@ -21,6 +22,8 @@ disable=abstract-method, cell-var-from-loop, comparison-with-callable, consider-iterating-dictionary, + consider-using-dict-items, + consider-using-from-import, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, @@ -105,6 +108,7 @@ disable=abstract-method, unsubscriptable-object, unused-argument, unused-import, + unused-private-member, unused-variable, unused-wildcard-import, used-before-assignment, From ee2df97bbdf9120ccefc9c132bcbf0994479f948 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 16 Jul 2021 17:28:32 +0500 Subject: [PATCH 1747/1853] Pin the libxml2 version in CI as a newer one breaks lxml (#5208) --- .github/workflows/tests-ubuntu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 521d7ae70..57188bd63 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -57,7 +57,8 @@ jobs: if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev + # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version + sudo apt-get install libxml2-dev/bionic-updates libxslt-dev - name: Run tests env: ${{ matrix.env }} From 70dddfe2b293171db4c58511175edf55bbe1831c Mon Sep 17 00:00:00 2001 From: Pascal Corpet Date: Wed, 21 Jul 2021 17:10:10 +0200 Subject: [PATCH 1748/1853] Typing: switch to a newer version of MyPy to check types --- tests/CrawlerProcess/asyncio_deferred_signal.py | 2 ++ tox.ini | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 46c2a12a4..bdd3c1fef 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,5 +1,6 @@ import asyncio import sys +from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -31,6 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: Optional[str] try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tox.ini b/tox.ini index 8167aff96..4c4bbff6e 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,10 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==0.780 + lxml-stubs==0.2.0 + mypy==0.910 + types-pyOpenSSL==20.0.3 + types-setuptools==57.0.0 commands = mypy --show-error-codes {posargs: scrapy tests} From 209c1fce02a776b2917559c09d977827f858743a Mon Sep 17 00:00:00 2001 From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com> Date: Sat, 24 Jul 2021 14:50:48 +1000 Subject: [PATCH 1749/1853] Reference MailSender in StatsMailer Added a reference to MailSender in the StatsMailer extension description and included a link to the document detailing how to instantiate MailSender and using Scrapy settings objects. --- docs/topics/extensions.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 9e86fd0fe..3cabcefdd 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -323,6 +323,15 @@ domain has finished scraping, including the Scrapy stats collected. The email will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` setting. +Emails can be sent using the MailSender class + +.. module:: scrapy.mail + :synopsis: MailSender class + +.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) + +To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email` + .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy From b22a0043988a4f3c54709988de99f489db44f78d Mon Sep 17 00:00:00 2001 From: Rob Banagale Date: Mon, 26 Jul 2021 11:51:32 -0700 Subject: [PATCH 1750/1853] Document media pipeline file naming (#5152) --- docs/topics/media-pipeline.rst | 88 ++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 3438cb637..46bd2859b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -111,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: IMAGES_STORE = '/path/to/valid/dir' +.. _topics-file-naming: + +File Naming +=========== + +Default File Naming +------------------- + +By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names. + +For example, the following image URL:: + + http://www.example.com/image.jpg + +Whose ``SHA-1 hash`` is:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a + +Will be downloaded and stored using your chosen :ref:`storage method ` and the following file name:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + +Custom File Naming +------------------- + +You may wish to use a different calculated file name for saved files. +For example, classifying an image by including meta in the file name. + +Customize file names by overriding the ``file_path`` method of your +media pipeline. + +For example, an image pipeline with image URL:: + + http://www.example.com/product/images/large/front/0000000004166 + +Can be processed into a file name with a condensed hash and the perspective +``front``:: + + 00b08510e4_front.jpg + +By overriding ``file_path`` like this: + +.. code-block:: python + + import hashlib + from os.path import splitext + + def file_path(self, request, response=None, info=None, *, item=None): + image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) + image_perspective = request.url.split('/')[-2] + image_filename = f'{image_url_hash}_{image_perspective}.jpg' + + return image_filename + +.. warning:: + If your custom file name scheme relies on meta data that can vary between + scrapes it may lead to unexpected re-downloading of existing media using + new file names. + + For example, if your custom file name scheme uses a product title and the + site changes an item's product title between scrapes, Scrapy will re-download + the same media using updated file names. + +For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`. + +.. _topics-supported-storage: + Supported Storage ================= File system storage ------------------- -The files are stored using a `SHA1 hash`_ of their URLs for the file names. +File system storage will save files to the following path:: -For example, the following image URL:: - - http://www.example.com/image.jpg - -Whose ``SHA1 hash`` is:: - - 3afec3b4765f8f0a07b78f98c07b83f013567a0a - -Will be downloaded and stored in the following file:: - - /full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + /full/ Where: @@ -139,6 +196,9 @@ Where: * ``full`` is a sub-directory to separate full images from thumbnails (if used). For more info see :ref:`topics-images-thumbnails`. +* ```` is the file name assigned to the file. For more info see :ref:`topics-file-naming`. + + .. _media-pipeline-ftp: FTP server storage @@ -353,9 +413,9 @@ Where: * ```` is the one specified in the :setting:`IMAGES_THUMBS` dictionary keys (``small``, ``big``, etc) -* ```` is the `SHA1 hash`_ of the image url +* ```` is the `SHA-1 hash`_ of the image url -.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions +.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions Example of image files stored using ``small`` and ``big`` thumbnail names:: From abe0b37d307d40897ca6d7e61aa5c137c8e6a4c1 Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Tue, 27 Jul 2021 17:11:32 +0800 Subject: [PATCH 1751/1853] Cleanup leftover boto2 code in S3DownloaderHandler (#5209) S3DownloaderHandler.conn is a leftover attribute from 5e99758. --- scrapy/core/downloader/handlers/s3.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1966570d4..31f1be31a 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,5 +1,3 @@ -from urllib.parse import unquote - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available @@ -59,7 +57,7 @@ class S3DownloadHandler: url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) - elif self._signer is not None: + else: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, @@ -69,14 +67,4 @@ class S3DownloadHandler: self._signer.add_auth(awsrequest) request = request.replace( url=url, headers=awsrequest.headers.items()) - else: - signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body, - ) - request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider) From 7e4321f201a795166d779f2aa0b36d38cb50106e Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Mon, 19 Jul 2021 12:00:42 +0800 Subject: [PATCH 1752/1853] Add support for temporary security credential in AWS auth --- docs/topics/feed-exports.rst | 5 +++-- docs/topics/settings.rst | 14 ++++++++++++++ scrapy/core/downloader/handlers/s3.py | 5 ++++- scrapy/extensions/feedexport.py | 5 ++++- scrapy/pipelines/files.py | 3 +++ scrapy/pipelines/images.py | 1 + tests/test_feedexport.py | 12 ++++++++---- tox.ini | 1 + 8 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 216a8bc52..af60de716 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -135,7 +135,7 @@ Here are some examples to illustrate: - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` -.. note:: :ref:`Spider arguments ` become spider attributes, hence +.. note:: :ref:`Spider arguments ` become spider attributes, hence they can also be used as storage URI parameters. @@ -200,6 +200,7 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_SESSION_TOKEN` (Optional) You can also define a custom ACL and custom endpoint for exported feeds using this setting: @@ -357,7 +358,7 @@ For instance:: 'item_export_kwargs': { 'export_empty_fields': True, }, - }, + }, '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1290b4a5e..58daafa6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -204,6 +204,20 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: AWS_SESSION_TOKEN + +AWS_SESSION_TOKEN +----------------- + +Default: ``None`` (Optional) + +The AWS security token used by code that requires access to `Amazon Web services`_, +such as the :ref:`S3 feed storage backend `. + +The security token is only required by a *temporary security credentials*. +Using of temporary security credentials is discouraged cause the credentials +are short term. It may expires before the scraping is done. + .. setting:: AWS_ENDPOINT_URL AWS_ENDPOINT_URL diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 31f1be31a..51ca1ed5e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -10,6 +10,7 @@ class S3DownloadHandler: def __init__(self, settings, *, crawler=None, aws_access_key_id=None, aws_secret_access_key=None, + aws_session_token=None, httpdownloadhandler=HTTPDownloadHandler, **kw): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -18,6 +19,8 @@ class S3DownloadHandler: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + if not aws_session_token: + aws_session_token = settings['AWS_SESSION_TOKEN'] # If no credentials could be found anywhere, # consider this an anonymous connection request by default; @@ -36,7 +39,7 @@ class S3DownloadHandler: if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) + aws_access_key_id, aws_secret_access_key, aws_session_token)) _http_handler = create_instance( objcls=httpdownloadhandler, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bd4808e2b..564c736f2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -154,13 +154,14 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, - feed_options=None): + feed_options=None, session_token=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key + self.session_token = session_token self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url @@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage): self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' @@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage): uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + session_token=crawler.settings['AWS_SESSION_TOKEN'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 13ecd4e6c..8766ef66f 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -79,6 +79,7 @@ class FSFilesStore: class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_SESSION_TOKEN = None AWS_ENDPOINT_URL = None AWS_REGION_NAME = None AWS_USE_SSL = None @@ -98,6 +99,7 @@ class S3FilesStore: 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + aws_session_token=self.AWS_SESSION_TOKEN, endpoint_url=self.AWS_ENDPOINT_URL, region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, @@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e3ab23ea5..9c99dc69e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index da0b2c786..389808306 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase): def test_parse_credentials(self): skip_if_no_boto() aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', - 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} + 'AWS_SECRET_ACCESS_KEY': 'settings_secret', + 'AWS_SESSION_TOKEN': 'settings_token'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler storage = S3FeedStorage.from_crawler( @@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase): ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # Instantiate directly storage = S3FeedStorage('s3://mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], - aws_credentials['AWS_SECRET_ACCESS_KEY']) + aws_credentials['AWS_SECRET_ACCESS_KEY'], + session_token=aws_credentials['AWS_SESSION_TOKEN']) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # URI priority > settings priority storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], @@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url): - super().__init__(uri, access_key, secret_key, acl, endpoint_url) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): + super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): diff --git a/tox.ini b/tox.ini index 4c4bbff6e..96050223b 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN GCS_TEST_FILE_URI GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools From 8e7b96d8a2a6d712be29773b4c69e190d0c1ac7b Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Tue, 27 Jul 2021 19:29:25 +0800 Subject: [PATCH 1753/1853] Tweak doc for setting AWS_SESSION_TOKEN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/feed-exports.rst | 4 +++- docs/topics/settings.rst | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index af60de716..2b3217d62 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -200,7 +200,9 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` -- :setting:`AWS_SESSION_TOKEN` (Optional) +- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_) + +.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys You can also define a custom ACL and custom endpoint for exported feeds using this setting: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 58daafa6f..1a1a833df 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -209,14 +209,13 @@ such as the :ref:`S3 feed storage backend `. AWS_SESSION_TOKEN ----------------- -Default: ``None`` (Optional) +Default: ``None`` The AWS security token used by code that requires access to `Amazon Web services`_, -such as the :ref:`S3 feed storage backend `. +such as the :ref:`S3 feed storage backend `, when using +`temporary security credentials`_. -The security token is only required by a *temporary security credentials*. -Using of temporary security credentials is discouraged cause the credentials -are short term. It may expires before the scraping is done. +.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys .. setting:: AWS_ENDPOINT_URL From d55b6fcad6a792c16022324c6dd6402f8fde8641 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 28 Jul 2021 12:10:34 +1000 Subject: [PATCH 1754/1853] Fix for duplicate object description error --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 3cabcefdd..08272a25d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -328,7 +328,7 @@ Emails can be sent using the MailSender class .. module:: scrapy.mail :synopsis: MailSender class -.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) +.. class:: MailSender To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email` From 494e0ad8ffc34e7d8079db6dd24fdc8265e81800 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:29:50 -0300 Subject: [PATCH 1755/1853] Update docs/topics/dynamic-content.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/dynamic-content.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 56c8b6ae9..9706f43fe 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -272,10 +272,10 @@ The following is a simple snippet to illustrate its usage within Scrapy:: For this example to work, Scrapy needs to be running on top of the :ref:`asyncio reactor `. -Keep in mind that this is just a proof of concept, since it circumvents -most of the Scrapy components (middlewares, dupefilter, etc). - -The following is a list of 3rd party projects which provide better integration: +Using pypeteer_ directly circumvents most of the +Scrapy components (middlewares, dupefilter, etc). Use +one of the following Scrapy plugins for better integration +with Scrapy: * https://github.com/elacuesta/scrapy-pyppeteer * https://github.com/lopuhin/scrapy-pyppeteer From 0e3d50dd186666362ab8358c9f89036917242c81 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:30:16 -0300 Subject: [PATCH 1756/1853] Update docs/topics/dynamic-content.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/dynamic-content.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 9706f43fe..e918bc006 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -252,6 +252,7 @@ Since version 2.0, it is possible to integrate libraries that use the ``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial Python port of `puppeteer`_), which uses headless Chrome to download and render pages. + The following is a simple snippet to illustrate its usage within Scrapy:: import pyppeteer From 4b62ac6c3ace093d16cf97e737689ac7942d52f9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 28 Jul 2021 15:00:24 -0300 Subject: [PATCH 1757/1853] Update headless browser docs to mention playwright --- docs/topics/dynamic-content.rst | 55 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index f96be0bbc..ea5d06210 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -246,55 +246,46 @@ Using a headless browser ======================== A `headless browser`_ is a special web browser that provides an API for -automation. +automation. By installing the :ref:`asyncio reactor `, +it is possible to integrate ``asyncio``-based libraries which handle headless browsers. -Since version 2.0, it is possible to integrate libraries that use the -``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial -Python port of `puppeteer`_), which uses headless Chrome to download and -render pages. +One such library is `playwright-python`_ (an official Python port of `playwright`_). +The following is a simple snippet to illustrate its usage within a Scrapy spider:: -The following is a simple snippet to illustrate its usage within Scrapy:: - - import pyppeteer import scrapy + from playwright.async_api import async_playwright - class PyppeteerSpider(scrapy.Spider): - name = "pyppeteer" - start_urls = ["data:,"] # avoid making an actual upstream request + class PlaywrightSpider(scrapy.Spider): + name = "playwright" + start_urls = ["data:,"] # avoid using the default Scrapy downloader async def parse(self, response): - browser = await pyppeteer.launch() - page = await browser.newPage() - await page.goto("https:/example.org") - title = await page.title() - await page.close() - yield {"title": title} + async with async_playwright() as pw: + browser = await pw.chromium.launch() + page = await browser.new_page() + await page.goto("https:/example.org") + title = await page.title() + return {"title": title} -For this example to work, Scrapy needs to be running on top of the -:ref:`asyncio reactor `. - -Using pypeteer_ directly circumvents most of the -Scrapy components (middlewares, dupefilter, etc). Use -one of the following Scrapy plugins for better integration -with Scrapy: - -* https://github.com/elacuesta/scrapy-pyppeteer -* https://github.com/lopuhin/scrapy-pyppeteer -* https://github.com/clemfromspace/scrapy-puppeteer +However, using `playwright-python`_ directly as in the above example +circumvents most of the Scrapy components (middlewares, dupefilter, etc). +We recommend using `scrapy-playwright`_ for a better integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 -.. _chompjs: https://github.com/Nykakin/chompjs .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets +.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript +.. _Splash: https://github.com/scrapinghub/splash +.. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.haxx.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser -.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _js2xml: https://github.com/scrapinghub/js2xml -.. _puppeteer: https://pptr.dev/ +.. _playwright-python: https://github.com/microsoft/playwright-python +.. _playwright: https://github.com/microsoft/playwright .. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract +.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash -.. _Splash: https://github.com/scrapinghub/splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ .. _wgrep: https://github.com/stav/wgrep From cc89f6be381d72a2528c8a672158671305019324 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 29 Jul 2021 17:12:44 -0300 Subject: [PATCH 1758/1853] Response.attributes (#5218) --- docs/topics/request-response.rst | 7 ++-- scrapy/http/response/__init__.py | 23 +++++++++---- scrapy/http/response/text.py | 8 ++--- tests/test_http_response.py | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a6a3daf31..d3e08efd4 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -670,9 +670,6 @@ Response objects .. autoclass:: Response - A :class:`Response` object represents an HTTP response, which is usually - downloaded (by the Downloader) and fed to the Spiders for processing. - :param url: the URL of this response :type url: str @@ -829,6 +826,8 @@ Response objects handlers, i.e. for ``http(s)`` responses. For other handlers, :attr:`protocol` is always ``None``. + .. autoattribute:: Response.attributes + .. method:: Response.copy() Returns a new Response which is a copy of this Response. @@ -925,6 +924,8 @@ TextResponse objects A :class:`~scrapy.Selector` instance using the response as target. The selector is lazily instantiated on first access. + .. autoattribute:: TextResponse.attributes + :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 185a9bb67..4de6c9b5b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from typing import Generator +from typing import Generator, Tuple from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -16,6 +16,19 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): + """An object that represents an HTTP response, which is usually + downloaded (by the Downloader) and fed to the Spiders for processing. + """ + + attributes: Tuple[str, ...] = ( + "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + + Currently used by :meth:`Response.replace`. + """ def __init__( self, @@ -97,12 +110,8 @@ class Response(object_ref): return self.replace() def replace(self, *args, **kwargs): - """Create a new Response with the same attributes except for those - given new values. - """ - for x in [ - "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", - ]: + """Create a new Response with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index e36e14880..27bd55c07 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import json import warnings from contextlib import suppress -from typing import Generator +from typing import Generator, Tuple from urllib.parse import urljoin import parsel @@ -30,6 +30,8 @@ class TextResponse(Response): _DEFAULT_ENCODING = 'ascii' _cached_decoded_json = _NONE + attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) self._cached_benc = None @@ -53,10 +55,6 @@ class TextResponse(Response): else: super()._set_body(body) - def replace(self, *args, **kwargs): - kwargs.setdefault('encoding', self.encoding) - return Response.replace(self, *args, **kwargs) - @property def encoding(self): return self._declared_encoding() or self._body_inferred_encoding() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 04a594d03..cf34a9e5c 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -820,3 +820,62 @@ class XmlResponseTest(TextResponseTest): response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), response.selector.xpath("//s2:elem/text()").getall(), ) + + +class CustomResponse(TextResponse): + attributes = TextResponse.attributes + ("foo", "bar") + + def __init__(self, *args, **kwargs) -> None: + self.foo = kwargs.pop("foo", None) + self.bar = kwargs.pop("bar", None) + self.lost = kwargs.pop("lost", None) + super().__init__(*args, **kwargs) + + +class CustomResponseTest(TextResponseTest): + response_class = CustomResponse + + def test_copy(self): + super().test_copy() + r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + r2 = r1.copy() + self.assertIsInstance(r2, self.response_class) + self.assertEqual(r1.foo, r2.foo) + self.assertEqual(r1.bar, r2.bar) + self.assertEqual(r1.lost, "lost") + self.assertIsNone(r2.lost) + + def test_replace(self): + super().test_replace() + r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + + r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") + self.assertIsInstance(r2, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r2.foo, "new-foo") + self.assertEqual(r2.bar, "new-bar") + self.assertEqual(r2.lost, "new-lost") + + r3 = r1.replace(foo="new-foo", bar="new-bar") + self.assertIsInstance(r3, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r3.foo, "new-foo") + self.assertEqual(r3.bar, "new-bar") + self.assertIsNone(r3.lost) + + r4 = r1.replace(foo="new-foo") + self.assertIsInstance(r4, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r4.foo, "new-foo") + self.assertEqual(r4.bar, "bar") + self.assertIsNone(r4.lost) + + with self.assertRaises(TypeError) as ctx: + r1.replace(unknown="unknown") + self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'") From 880a4d9493338516aa6d12d602dadf9be60d3053 Mon Sep 17 00:00:00 2001 From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com> Date: Sun, 1 Aug 2021 11:02:27 +1000 Subject: [PATCH 1759/1853] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 08272a25d..297e1fdc5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -323,14 +323,10 @@ domain has finished scraping, including the Scrapy stats collected. The email will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` setting. -Emails can be sent using the MailSender class - -.. module:: scrapy.mail - :synopsis: MailSender class - -.. class:: MailSender - -To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email` +Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a +full list of parameters, including examples on how to instantiate +:class:`~scrapy.mail.MailSender` and use mail settings, see +:ref:`topics-email`. .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy From 2bf2f9d6db89968bcb5df6bc7d093fdd53de5ba5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Aug 2021 19:44:11 +0500 Subject: [PATCH 1760/1853] Add Python 3.10b4 tests on Ubuntu. --- .github/workflows/tests-ubuntu.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 57188bd63..57c994158 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -45,6 +45,14 @@ jobs: env: TOXENV: asyncio + # 3.10-pre + - python-version: "3.10.0-beta.4" + env: + TOXENV: py + - python-version: "3.10.0-beta.4" + env: + TOXENV: asyncio + steps: - uses: actions/checkout@v2 @@ -54,7 +62,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version From ef6fb933b568c497ab3745284bc2a02725bced1c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Jul 2021 12:02:15 +0500 Subject: [PATCH 1761/1853] Fix a Python 3.10 logging issue. --- scrapy/utils/signal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 62808f3ce..fbafc9d45 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" -import collections +import collections.abc import logging from twisted.internet.defer import DeferredList, Deferred @@ -21,7 +21,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): Failures instead of exceptions. """ dont_log = named.pop('dont_log', ()) - dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,) dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] From 93bf1ae7e3966d56539411c59d172c2971ee61e0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Aug 2021 20:16:29 +0500 Subject: [PATCH 1762/1853] Fix tests for the 3.10 TypeError message change. --- tests/test_http_response.py | 2 +- tests/test_request_cb_kwargs.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cf34a9e5c..c376a46cd 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -878,4 +878,4 @@ class CustomResponseTest(TextResponseTest): with self.assertRaises(TypeError) as ctx: r1.replace(unknown="unknown") - self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'") + self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'")) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 145a4e9b2..b68184b87 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -158,12 +158,14 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_less'].exc_info[1]), - "parse_takes_less() got an unexpected keyword argument 'number'" + self.assertTrue( + str(exceptions['takes_less'].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ) ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_more'].exc_info[1]), - "parse_takes_more() missing 1 required positional argument: 'other'" + self.assertTrue( + str(exceptions['takes_more'].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ) ) From 94baa4b27273e5a779bb977cea4c8eb5301fc3bf Mon Sep 17 00:00:00 2001 From: Mannan2812 <42071936+Mannan2812@users.noreply.github.com> Date: Fri, 6 Aug 2021 00:53:11 +0530 Subject: [PATCH 1763/1853] Fix FileFeedStoragePreFeedOptionsTest fails in CI/CD pipeline (#5198) --- tests/test_feedexport.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 389808306..53e6a2018 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1932,14 +1932,15 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): maxDiff = None def test_init(self): - settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': FileFeedStorageWithoutFeedOptions - }, - } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with tempfile.NamedTemporaryFile() as temp: + settings_dict = { + 'FEED_URI': f'file:///{temp.name}', + 'FEED_STORAGES': { + 'file': FileFeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") with warnings.catch_warnings(record=True) as w: feed_exporter.open_spider(spider) From 8e7d2ef13312bfb4ec5e1800f00108806ddc12e8 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Sat, 7 Aug 2021 11:44:12 +1000 Subject: [PATCH 1764/1853] Document JOBDIR option issue #5173 Add JOBDIR setting to the settings page. Add default JOBDIR setting to global defaults in scrapy.settings.default_settings module. --- docs/topics/settings.rst | 11 +++++++++++ scrapy/settings/default_settings.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1a1a833df..5e820b0a9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -989,6 +989,17 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. +.. setting:: JOBDIR + +JOBDIR +------ + +Default: ``''`` + +A string indicating the directory for storing the required data to keep the state of a single job to enable persistence support. This directory must not be shared by different spiders or jobs/runs of the same spider. + +For more info on this setting, see :ref:`topics-jobs` + .. setting:: LOG_ENABLED LOG_ENABLED diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4ef330dd2..9137086c0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -199,6 +199,8 @@ ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +JOBDIR = '' + LOG_ENABLED = True LOG_ENCODING = 'utf-8' LOG_FORMATTER = 'scrapy.logformatter.LogFormatter' From 48eff4ee8f21535e98baf2bdff91749fee002c10 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Sun, 8 Aug 2021 20:52:14 +1000 Subject: [PATCH 1765/1853] Remove JOBDIR from default settings --- scrapy/settings/default_settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9137086c0..4ef330dd2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -199,8 +199,6 @@ ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} -JOBDIR = '' - LOG_ENABLED = True LOG_ENCODING = 'utf-8' LOG_FORMATTER = 'scrapy.logformatter.LogFormatter' From 954f3035908f7f7f528ce4d3c5245f056645dc7f Mon Sep 17 00:00:00 2001 From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com> Date: Mon, 9 Aug 2021 22:23:23 +1000 Subject: [PATCH 1766/1853] Update docs/topics/settings.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5e820b0a9..2ab2020fa 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -996,9 +996,8 @@ JOBDIR Default: ``''`` -A string indicating the directory for storing the required data to keep the state of a single job to enable persistence support. This directory must not be shared by different spiders or jobs/runs of the same spider. - -For more info on this setting, see :ref:`topics-jobs` +A string indicating the directory for storing the state of a crawl when +:ref:`pausing and resuming crawls `. .. setting:: LOG_ENABLED From 1ba0f68483cfb8aa62759e21b3479ec9bea94beb Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 17:09:37 +0200 Subject: [PATCH 1767/1853] Allow comma-separated values in the rel tag Comma-separated `rel` values are often seen in the wild, because Google allows it (see https://developers.google.com/search/docs/advanced/guidelines/qualify-outbound-links). --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5c986eedc..51cef1e91 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -138,7 +138,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return rel is not None and 'nofollow' in rel.split() + return rel is not None and 'nofollow' in rel.replace(',', ' ').split() def create_instance(objcls, settings, crawler, *args, **kwargs): From 18b6f30a7359d1a798c30888ddd10a1612d8e711 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:13:50 +0200 Subject: [PATCH 1768/1853] Add test for rel_has_nofollow --- tests/test_utils_misc/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e95a3a316..67367dbfb 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from unittest import mock from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules __doctests__ = ['scrapy.utils.misc'] @@ -162,6 +162,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') == 'test' + def test_rel_has_nofollow(self): + assert os.environ.get('some_test_environ') is None + asert rel_has_nofollow('ugc nofollow') == True + asert rel_has_nofollow('ugc,nofollow') == True + asert rel_has_nofollow('ugc') == False + if __name__ == "__main__": unittest.main() From 07d20a8ce45ab0cbf61d08214db4963302661257 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:21:43 +0200 Subject: [PATCH 1769/1853] Fix test_rel_has_nofollow test --- tests/test_utils_misc/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 67367dbfb..b0d7acd12 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -163,10 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test' def test_rel_has_nofollow(self): - assert os.environ.get('some_test_environ') is None - asert rel_has_nofollow('ugc nofollow') == True - asert rel_has_nofollow('ugc,nofollow') == True - asert rel_has_nofollow('ugc') == False + assert rel_has_nofollow('ugc nofollow') == True + assert rel_has_nofollow('ugc,nofollow') == True + assert rel_has_nofollow('ugc') == False if __name__ == "__main__": From 295f0e2bf5c352c6ddf27a188af10bf122d1c6b0 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:38:29 +0200 Subject: [PATCH 1770/1853] Make flake8 happy --- tests/test_utils_misc/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index b0d7acd12..69f593ccd 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -163,9 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test' def test_rel_has_nofollow(self): - assert rel_has_nofollow('ugc nofollow') == True - assert rel_has_nofollow('ugc,nofollow') == True - assert rel_has_nofollow('ugc') == False + assert rel_has_nofollow('ugc nofollow') is True + assert rel_has_nofollow('ugc,nofollow') is True + assert rel_has_nofollow('ugc') is False if __name__ == "__main__": From ce9d6c658b21a5d9d9605a2683b7a143f2077dfa Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 22:21:51 +0200 Subject: [PATCH 1771/1853] Add more rel_has_nofollow tests --- tests/test_utils_misc/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 69f593ccd..47d73a2dd 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -166,6 +166,10 @@ class UtilsMiscTestCase(unittest.TestCase): assert rel_has_nofollow('ugc nofollow') is True assert rel_has_nofollow('ugc,nofollow') is True assert rel_has_nofollow('ugc') is False + assert rel_has_nofollow('nofollow') is True + assert rel_has_nofollow('nofollowfoo') is False + assert rel_has_nofollow('foonofollow') is False + assert rel_has_nofollow('ugc, , nofollow') is True if __name__ == "__main__": From 983b89ad4f72730c37b32c9240a697a4c1f24183 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 10:39:23 +0500 Subject: [PATCH 1772/1853] Fix SpiderLoaderTest on Python 3.10. --- tests/test_spiderloader/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 4929f1e3e..8a35e9fd7 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -118,6 +118,11 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({'SPIDER_MODULES': [module], 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() From 74cee38a4e07c31a8dd2a8772ff18d1b9adf8a6b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 14:19:08 +0500 Subject: [PATCH 1773/1853] Don't run the asyncio tests on 3.9. --- .github/workflows/tests-ubuntu.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 57c994158..81beda5da 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -41,9 +41,6 @@ jobs: env: TOXENV: extra-deps TOX_PIP_VERSION: 20.3.3 - - python-version: 3.9 - env: - TOXENV: asyncio # 3.10-pre - python-version: "3.10.0-beta.4" From b63369c148b9a1f87a974e21c5e307a62783d6fd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 20:02:45 +0500 Subject: [PATCH 1774/1853] Rename tests/requirements-py3.txt to tests/requirements.txt. --- tests/{requirements-py3.txt => requirements.txt} | 0 tox.ini | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/{requirements-py3.txt => requirements.txt} (100%) diff --git a/tests/requirements-py3.txt b/tests/requirements.txt similarity index 100% rename from tests/requirements-py3.txt rename to tests/requirements.txt diff --git a/tox.ini b/tox.ini index 96050223b..e274fc8d2 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ minversion = 1.7.0 [testenv] deps = - -rtests/requirements-py3.txt + -rtests/requirements.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe @@ -82,7 +82,7 @@ deps = Twisted[http2]==17.9.0 w3lib==1.17.0 zope.interface==4.1.3 - -rtests/requirements-py3.txt + -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment From 2814e0e1972fa38151b6800c881d49f50edf9c6b Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 16 Aug 2021 16:22:01 +0500 Subject: [PATCH 1775/1853] Disable builtin middlewares in spider middleware tests. (#5229) --- tests/test_spidermiddleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 78e926adc..b39576996 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -15,7 +15,7 @@ class SpiderMiddlewareTestCase(TestCase): def setUp(self): self.request = Request('http://example.com/index.html') self.response = Response(self.request.url, request=self.request) - self.crawler = get_crawler(Spider) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}}) self.spider = self.crawler._create_spider('foo') self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) From 8bbaea9892003769672204a8ff4e989b5aab5ceb Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 16:57:43 +0530 Subject: [PATCH 1776/1853] updated documentation for python version for reppy --- docs/topics/downloader-middleware.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 80c6c2c37..222dda685 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1073,6 +1073,8 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` +* only works with python 3.8 and earlier + .. _rerp-parser: Robotexclusionrulesparser From 1a8b98843aee548a52faa36f5360a81a1624e208 Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 17:00:05 +0530 Subject: [PATCH 1777/1853] updated documentation for python version for reppy --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 222dda685..2c00ad45d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1073,7 +1073,7 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -* only works with python 3.8 and earlier +* Only works with python 3.8 and earlier .. _rerp-parser: From cc1cb2de0c6e91393ceb6872c174aa2ac06c07ac Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 17:21:47 +0530 Subject: [PATCH 1778/1853] updated suggested changes --- docs/topics/downloader-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2c00ad45d..fa211fb75 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,12 +1068,13 @@ Native implementation, provides better speed than Protego. In order to use this parser: +.. warning:: Does not support Python 3.9+ + * Install `Reppy `_ by running ``pip install reppy`` * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -* Only works with python 3.8 and earlier .. _rerp-parser: From 013ac90f6129a9e8b862d66ca2ffa8f0f1fd674e Mon Sep 17 00:00:00 2001 From: umair ansari Date: Mon, 16 Aug 2021 18:00:06 +0530 Subject: [PATCH 1779/1853] Update docs/topics/downloader-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fa211fb75..8323bc564 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego. In order to use this parser: -.. warning:: Does not support Python 3.9+ - * Install `Reppy `_ by running ``pip install reppy`` + .. warning:: Does not support Python 3.9+ + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From ebddb77a331c6290e64e449ef9847e33b323a146 Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 18:08:26 +0530 Subject: [PATCH 1780/1853] updated suggested changes after review --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fa211fb75..4d7c87404 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego. In order to use this parser: -.. warning:: Does not support Python 3.9+ - * Install `Reppy `_ by running ``pip install reppy`` +.. warning:: Does not support Python 3.9+ + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From bcf38a67194f25db66334af18bdf49b34c6a0c39 Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 14:48:47 +0530 Subject: [PATCH 1781/1853] added upstream issue for not supported python version --- docs/topics/downloader-middleware.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 4d7c87404..089d5683a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,7 +1070,9 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` -.. warning:: Does not support Python 3.9+ + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in + Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From d623ed15d1a79a91b55aa7aae2d942aac94abfdf Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 14:51:03 +0530 Subject: [PATCH 1782/1853] indentation updated --- docs/topics/downloader-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 089d5683a..928a59bf1 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,9 +1070,9 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` - .. warning:: `Upstream issue #122 - `_ prevents reppy usage in - Python 3.9+. +.. warning:: `Upstream issue #122 + `_ prevents reppy usage in + Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From 2d2581c68f35799dc4372a257eaa8dbb5208481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Aug 2021 12:46:42 +0200 Subject: [PATCH 1783/1853] Move documentation about avoiding bans into a topic of its own (#4039) --- docs/index.rst | 4 + docs/topics/avoiding-bans.rst | 340 ++++++++++++++++++++++++++++++++++ docs/topics/practices.rst | 34 ---- 3 files changed, 344 insertions(+), 34 deletions(-) create mode 100644 docs/topics/avoiding-bans.rst diff --git a/docs/index.rst b/docs/index.rst index 433798aa8..7647b3781 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,6 +155,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -179,6 +180,9 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/avoiding-bans` + Avoid getting banned from websites. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst new file mode 100644 index 000000000..59f0da191 --- /dev/null +++ b/docs/topics/avoiding-bans.rst @@ -0,0 +1,340 @@ +.. _bans: + +============= +Avoiding bans +============= + +This topic covers some of the strategies that you can follow to avoid getting +different or bad responses from a website that you are crawling due to filters +such as regional filters, web browser filters, etc. + +.. _avoiding-crawls: + +Avoiding crawls +=============== + +The best way not to be banned from a website is not to send requests to it in +the first place. + +One way to avoid crawling a website is to find the desired dataset through +other means. For example, you can use Google’s `dataset search engine`_. + +If the target website is the only or best source of the desired information, +and you only need to extract the data on a monthly basis or a lower frequency, +you may be able to crawl a public snapshot of the target website instead. +`Common Crawl`_ is an open repository of web crawl data that you can access +freely. It contains monthly snapshots of a wide variety of websites and, if you +are lucky, your target website will be among them. + +.. _Common Crawl: https://commoncrawl.org/ +.. _dataset search engine: https://datasetsearch.research.google.com/ + + +.. _being-polite: + +Being polite +============ + +To avoid being banned, you should first avoid giving a website reasons to ban +you. + +.. _identifying-yourself: + +Identifying yourself +-------------------- + +If your crawling has a noticeable negative impact on a website or you crawl +content that should not be crawled, website administrators will need to do +something. + +Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and +includes contact information, so that website administrators can contact you. + + +.. _following-robotstxt: + +Following robots.txt guidelines +------------------------------- + +Some websites provide a ``robots.txt`` file at their root path (e.g. +``http://example.com/robots.txt``) that describes the guidelines that they wish +bots to follow when crawling their website. + +Before you start writing a spider for a website, read their ``robots.txt`` +file and implement your spider following its guidelines. See the `robots.txt +standard draft`_ or the `robots.txt Google specification`_ for information on +how to read ``robots.txt`` files. + +To ensure that your spider does not crawl pages restricted by ``robots.txt`` +guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the +:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` +middleware. When you do, if your spider attempts to crawl a restricted page, +this middleware aborts that request with the following message:: + + Forbidden by robots.txt + +Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and +:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or +``Request-Rate`` directives from the ``robots.txt`` guidelines. + +You may also use the :ref:`AutoThrottle extension ` on top +of that, so that when the target website experiences a high load, your spider +automatically switches to higher download delays. + +.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt +.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 + + +.. _choosing-crawl-speed: + +Finding the right guidelines on your own +---------------------------------------- + +If a website does not specify a desired download delay, or does not provide a +``robots.txt`` file, you should make an effort to find out the right values for +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that +will not have a noticeable negative impact on the target website. + +Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that +the target website receives, and choose concurrency and delay values that will +not cause a noticeable traffic increase. + +.. _SimilarWeb: https://www.similarweb.com + + +.. _filters-and-challenges: + +Bypassing filters and solving challenges +======================================== + +Some websites implement filters and challenges that aim to deny access or alter +their content based on aspects of the visitor, such as the country where they +are or the web browsing tool they use. + +.. _regional-filter: + +Bypassing regional filters +-------------------------- + +Some websites send different or bad responses based on the region or country +associated to your `IP address`_. + +To bypass these filters, get access to a `proxy server`_ that has an outgoing +IP address from a region that gets the desired responses. + +Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` +middleware to configure your spider to use that proxy. + +.. _IP address: https://en.wikipedia.org/wiki/IP_address +.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server + + +.. _web-browser-filter: + +Bypassing web browser filters +----------------------------- + +Some websites send different or bad responses if they detect that your request +does not come from a web browser. + +To bypass these filters, switch your :setting:`USER_AGENT` to a value copied +from those that popular web browsers use. In some rare cases, you may need a +user agent string from a specific web browser. + +There are multiple Scrapy plugins that can rotate your requests through popular +web browser user agent strings, such as scrapy-fake-useragent_, +scrapy-random-useragent_ or Scrapy-UserAgents_. + +For advanced web browser filters, +:ref:`pre-rendering JavaScript ` or +:ref:`using a headless browser ` may be necessary. +Use these options only as a last resort, however, because they cause a higher +load per request on the target website. + +.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent +.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent +.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ + + +.. _request-delay-filter: + +Bypassing request delay filters +------------------------------- + +Some websites may ban your IP after they detect that your requests use a +constant download delay. + +To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting is enabled by default. When that is not enough, an +:ref:`IP address rotation solution ` may be much more effective. + + +.. _isp-filter: + +Bypassing internet service provider filters +------------------------------------------- + +Some websites send different or bad responses if they detect that your request +comes from an IP address that belongs to a `data center`_, as opposed to a +residential IP address from an `internet service provider`_ or a mobile IP +address from a `mobile network`_. + +To bypass these filters, get access to a proxy server that has an outgoing IP +address that is either residential or mobile. Note that you may also get +different responses depending on whether your IP address is residential or +mobile. + +Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` +middleware to configure your spider to use that proxy. + +.. _data center: https://en.wikipedia.org/wiki/Data_center +.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider +.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network + + +.. _captcha: + +Solving CAPTCHA challenges +-------------------------- + +Some websites require you to solve a `CAPTCHA challenge`_ to get the desired +response. + +To bypass these filters, several options exist: + +- You could have your spider present the CAPTCHA challenge to you and wait + for you to solve it manually. + +- Some CAPTCHA challenges can be solved using an `optical character + recognition`_ (OCR) solution such as pytesseract_. + +- Paid CAPTCHA solving services exist. + +Whichever solution you choose, implement it as a :ref:`downloader middleware +` that automatically detects CAPTCHA challenges +in responses and solves them, so that your spider code only receives successful +responses. + +.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA +.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition +.. _pytesseract: https://github.com/madmaze/pytesseract + + +.. _ip-rotation: + +IP address rotation solutions +============================= + +See below some of the different solutions there are to have your requests use +different outgoing IP addresses. + +When using this approach, remember to set :setting:`COOKIES_ENABLED` to +``False`` to disable global cookie handling. This prevents websites from +identifying two requests as coming from the same user agent even if they come +from different IP addresses and have different user-agent strings. You can +still include some cookies manually in your requests. Define them through the +``Cookies`` header of your requests. See +:class:`Request.headers `. + +.. _smart-proxy: + +Smart proxies +------------- + +An increasing number of websites use solutions that apply many of the above +filters and challenges at the same time. + +There are paid proxy services, like `Zyte Smart Proxy Manager`_, that +automatically bypass website filters and challenges, so that your spider only +gets successful responses. They also allow managing sessions to simulate user +behavior. + +For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced +integration with Scrapy. For other services, use the +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware +or implement your own :ref:`downloader middleware +`. + +.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ + + +.. _rotating-proxy: + +Rotating proxies +---------------- + +Rotating proxy services like ProxyMesh_ send different requests through +different proxies. This can decrease the likelihood of being affected by some +filters or challenges. + +.. _ProxyMesh: https://proxymesh.com/ + + +.. _free-proxies: + +Free proxies +------------ + +You can easily find lists of free proxies in the internet, and you can use +a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in +your spider and have requests rotate through them automatically. + +This approach, however, has serious drawbacks: + +- Free proxies may stop working at any moment. You need to implement a way to + refresh your list of free proxies. + +- In addition to handling occasional bad responses from websites, you + need to handle all kinds of bad responses from proxies. You may even need + to inspect the response body to determine if a response comes from the + target website or from a misbehaving proxy. + +- Advanced antibot solutions may automatically detect and filter out traffic + from free proxies. + +.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies + + +.. _custom-rotating-proxy: + +Custom rotating proxy +--------------------- + +If you have spare servers, you can set them up as proxies and use scrapoxy_ to +build a custom proxy that rotates traffic through them. However, the initial +setup can be complex, and your requests will be vulnerable to +:ref:`internet service provider filtering `. + +.. _scrapoxy: https://scrapoxy.io/ + + +.. _tor: + +The Tor network +--------------- + +It is possible to send requests through the `Tor network`_. + +The initial setup to have Scrapy working with Tor is not straightforward. +Use a search engine to find up-to-date documentation specific to using +Scrapy and Tor together. + +The main drawback of using the Tor network is that traffic can be extremely +slow. + +.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) + + +.. _commercial-support: + +Seeking professional help +========================= + +Avoiding bans, filters and challenges can be difficult and tricky, and may +sometimes require special infrastructure. + +If you find yourself unable to prevent your spider from getting bad responses, +consider contacting `commercial support`_. + +.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..a7a6fd129 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -226,39 +226,5 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 -.. _bans: -Avoiding getting banned -======================= - -Some websites implement certain measures to prevent bots from crawling them, -with varying degrees of sophistication. Getting around those measures can be -difficult and tricky, and may sometimes require special infrastructure. Please -consider contacting `commercial support`_ if in doubt. - -Here are some tips to keep in mind when dealing with these kinds of sites: - -* rotate your user agent from a pool of well-known ones from browsers (google - around to get a list of them) -* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use - cookies to spot bot behaviour -* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. -* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites - directly -* use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a - super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ - -If you are still unable to prevent your bot getting banned, consider contacting -`commercial support`_. - -.. _Tor project: https://www.torproject.org/ -.. _commercial support: https://scrapy.org/support/ -.. _ProxyMesh: https://proxymesh.com/ -.. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 572d347b3bc0149042f04ea83aff4a4f8fc7a831 Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 16:17:52 +0530 Subject: [PATCH 1784/1853] warning view updated --- docs/topics/downloader-middleware.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 928a59bf1..99d57bda9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,9 +1070,8 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` -.. warning:: `Upstream issue #122 - `_ prevents reppy usage in - Python 3.9+. + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From cd17c829cf0d7a006ab5594d56fe182a0ffc71d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Aug 2021 19:55:35 +0500 Subject: [PATCH 1785/1853] Revert "Move documentation about avoiding bans into a topic of its own (#4039)" This reverts commit 2d2581c68f35799dc4372a257eaa8dbb5208481d. --- docs/index.rst | 4 - docs/topics/avoiding-bans.rst | 340 ---------------------------------- docs/topics/practices.rst | 34 ++++ 3 files changed, 34 insertions(+), 344 deletions(-) delete mode 100644 docs/topics/avoiding-bans.rst diff --git a/docs/index.rst b/docs/index.rst index 7647b3781..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,7 +155,6 @@ Solving specific problems topics/debug topics/contracts topics/practices - topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -180,9 +179,6 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. -:doc:`topics/avoiding-bans` - Avoid getting banned from websites. - :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst deleted file mode 100644 index 59f0da191..000000000 --- a/docs/topics/avoiding-bans.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. _bans: - -============= -Avoiding bans -============= - -This topic covers some of the strategies that you can follow to avoid getting -different or bad responses from a website that you are crawling due to filters -such as regional filters, web browser filters, etc. - -.. _avoiding-crawls: - -Avoiding crawls -=============== - -The best way not to be banned from a website is not to send requests to it in -the first place. - -One way to avoid crawling a website is to find the desired dataset through -other means. For example, you can use Google’s `dataset search engine`_. - -If the target website is the only or best source of the desired information, -and you only need to extract the data on a monthly basis or a lower frequency, -you may be able to crawl a public snapshot of the target website instead. -`Common Crawl`_ is an open repository of web crawl data that you can access -freely. It contains monthly snapshots of a wide variety of websites and, if you -are lucky, your target website will be among them. - -.. _Common Crawl: https://commoncrawl.org/ -.. _dataset search engine: https://datasetsearch.research.google.com/ - - -.. _being-polite: - -Being polite -============ - -To avoid being banned, you should first avoid giving a website reasons to ban -you. - -.. _identifying-yourself: - -Identifying yourself --------------------- - -If your crawling has a noticeable negative impact on a website or you crawl -content that should not be crawled, website administrators will need to do -something. - -Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and -includes contact information, so that website administrators can contact you. - - -.. _following-robotstxt: - -Following robots.txt guidelines -------------------------------- - -Some websites provide a ``robots.txt`` file at their root path (e.g. -``http://example.com/robots.txt``) that describes the guidelines that they wish -bots to follow when crawling their website. - -Before you start writing a spider for a website, read their ``robots.txt`` -file and implement your spider following its guidelines. See the `robots.txt -standard draft`_ or the `robots.txt Google specification`_ for information on -how to read ``robots.txt`` files. - -To ensure that your spider does not crawl pages restricted by ``robots.txt`` -guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the -:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` -middleware. When you do, if your spider attempts to crawl a restricted page, -this middleware aborts that request with the following message:: - - Forbidden by robots.txt - -Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and -:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or -``Request-Rate`` directives from the ``robots.txt`` guidelines. - -You may also use the :ref:`AutoThrottle extension ` on top -of that, so that when the target website experiences a high load, your spider -automatically switches to higher download delays. - -.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt -.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 - - -.. _choosing-crawl-speed: - -Finding the right guidelines on your own ----------------------------------------- - -If a website does not specify a desired download delay, or does not provide a -``robots.txt`` file, you should make an effort to find out the right values for -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that -will not have a noticeable negative impact on the target website. - -Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that -the target website receives, and choose concurrency and delay values that will -not cause a noticeable traffic increase. - -.. _SimilarWeb: https://www.similarweb.com - - -.. _filters-and-challenges: - -Bypassing filters and solving challenges -======================================== - -Some websites implement filters and challenges that aim to deny access or alter -their content based on aspects of the visitor, such as the country where they -are or the web browsing tool they use. - -.. _regional-filter: - -Bypassing regional filters --------------------------- - -Some websites send different or bad responses based on the region or country -associated to your `IP address`_. - -To bypass these filters, get access to a `proxy server`_ that has an outgoing -IP address from a region that gets the desired responses. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _IP address: https://en.wikipedia.org/wiki/IP_address -.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server - - -.. _web-browser-filter: - -Bypassing web browser filters ------------------------------ - -Some websites send different or bad responses if they detect that your request -does not come from a web browser. - -To bypass these filters, switch your :setting:`USER_AGENT` to a value copied -from those that popular web browsers use. In some rare cases, you may need a -user agent string from a specific web browser. - -There are multiple Scrapy plugins that can rotate your requests through popular -web browser user agent strings, such as scrapy-fake-useragent_, -scrapy-random-useragent_ or Scrapy-UserAgents_. - -For advanced web browser filters, -:ref:`pre-rendering JavaScript ` or -:ref:`using a headless browser ` may be necessary. -Use these options only as a last resort, however, because they cause a higher -load per request on the target website. - -.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent -.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent -.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ - - -.. _request-delay-filter: - -Bypassing request delay filters -------------------------------- - -Some websites may ban your IP after they detect that your requests use a -constant download delay. - -To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting is enabled by default. When that is not enough, an -:ref:`IP address rotation solution ` may be much more effective. - - -.. _isp-filter: - -Bypassing internet service provider filters -------------------------------------------- - -Some websites send different or bad responses if they detect that your request -comes from an IP address that belongs to a `data center`_, as opposed to a -residential IP address from an `internet service provider`_ or a mobile IP -address from a `mobile network`_. - -To bypass these filters, get access to a proxy server that has an outgoing IP -address that is either residential or mobile. Note that you may also get -different responses depending on whether your IP address is residential or -mobile. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _data center: https://en.wikipedia.org/wiki/Data_center -.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider -.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network - - -.. _captcha: - -Solving CAPTCHA challenges --------------------------- - -Some websites require you to solve a `CAPTCHA challenge`_ to get the desired -response. - -To bypass these filters, several options exist: - -- You could have your spider present the CAPTCHA challenge to you and wait - for you to solve it manually. - -- Some CAPTCHA challenges can be solved using an `optical character - recognition`_ (OCR) solution such as pytesseract_. - -- Paid CAPTCHA solving services exist. - -Whichever solution you choose, implement it as a :ref:`downloader middleware -` that automatically detects CAPTCHA challenges -in responses and solves them, so that your spider code only receives successful -responses. - -.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA -.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition -.. _pytesseract: https://github.com/madmaze/pytesseract - - -.. _ip-rotation: - -IP address rotation solutions -============================= - -See below some of the different solutions there are to have your requests use -different outgoing IP addresses. - -When using this approach, remember to set :setting:`COOKIES_ENABLED` to -``False`` to disable global cookie handling. This prevents websites from -identifying two requests as coming from the same user agent even if they come -from different IP addresses and have different user-agent strings. You can -still include some cookies manually in your requests. Define them through the -``Cookies`` header of your requests. See -:class:`Request.headers `. - -.. _smart-proxy: - -Smart proxies -------------- - -An increasing number of websites use solutions that apply many of the above -filters and challenges at the same time. - -There are paid proxy services, like `Zyte Smart Proxy Manager`_, that -automatically bypass website filters and challenges, so that your spider only -gets successful responses. They also allow managing sessions to simulate user -behavior. - -For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced -integration with Scrapy. For other services, use the -:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware -or implement your own :ref:`downloader middleware -`. - -.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ - - -.. _rotating-proxy: - -Rotating proxies ----------------- - -Rotating proxy services like ProxyMesh_ send different requests through -different proxies. This can decrease the likelihood of being affected by some -filters or challenges. - -.. _ProxyMesh: https://proxymesh.com/ - - -.. _free-proxies: - -Free proxies ------------- - -You can easily find lists of free proxies in the internet, and you can use -a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in -your spider and have requests rotate through them automatically. - -This approach, however, has serious drawbacks: - -- Free proxies may stop working at any moment. You need to implement a way to - refresh your list of free proxies. - -- In addition to handling occasional bad responses from websites, you - need to handle all kinds of bad responses from proxies. You may even need - to inspect the response body to determine if a response comes from the - target website or from a misbehaving proxy. - -- Advanced antibot solutions may automatically detect and filter out traffic - from free proxies. - -.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies - - -.. _custom-rotating-proxy: - -Custom rotating proxy ---------------------- - -If you have spare servers, you can set them up as proxies and use scrapoxy_ to -build a custom proxy that rotates traffic through them. However, the initial -setup can be complex, and your requests will be vulnerable to -:ref:`internet service provider filtering `. - -.. _scrapoxy: https://scrapoxy.io/ - - -.. _tor: - -The Tor network ---------------- - -It is possible to send requests through the `Tor network`_. - -The initial setup to have Scrapy working with Tor is not straightforward. -Use a search engine to find up-to-date documentation specific to using -Scrapy and Tor together. - -The main drawback of using the Tor network is that traffic can be extremely -slow. - -.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) - - -.. _commercial-support: - -Seeking professional help -========================= - -Avoiding bans, filters and challenges can be difficult and tricky, and may -sometimes require special infrastructure. - -If you find yourself unable to prevent your spider from getting bad responses, -consider contacting `commercial support`_. - -.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a7a6fd129..732eba587 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -226,5 +226,39 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 +.. _bans: +Avoiding getting banned +======================= + +Some websites implement certain measures to prevent bots from crawling them, +with varying degrees of sophistication. Getting around those measures can be +difficult and tricky, and may sometimes require special infrastructure. Please +consider contacting `commercial support`_ if in doubt. + +Here are some tips to keep in mind when dealing with these kinds of sites: + +* rotate your user agent from a pool of well-known ones from browsers (google + around to get a list of them) +* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use + cookies to spot bot behaviour +* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. +* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites + directly +* use a pool of rotating IPs. For example, the free `Tor project`_ or paid + services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a + super proxy that you can attach your own proxies to. +* use a highly distributed downloader that circumvents bans internally, so you + can just focus on parsing clean pages. One example of such downloaders is + `Zyte Smart Proxy Manager`_ + +If you are still unable to prevent your bot getting banned, consider contacting +`commercial support`_. + +.. _Tor project: https://www.torproject.org/ +.. _commercial support: https://scrapy.org/support/ +.. _ProxyMesh: https://proxymesh.com/ +.. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders +.. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 3f635eb683821667b7a46a99531a95a8c05b6e1b Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Tue, 24 Aug 2021 12:05:50 +0300 Subject: [PATCH 1786/1853] Extract domain from genspider URL (#4439) --- scrapy/commands/genspider.py | 12 +++++++++++- tests/test_commands.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 5f44daa70..2082a4974 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,6 +4,7 @@ import string from importlib import import_module from os.path import join, dirname, abspath, exists, splitext +from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand @@ -22,6 +23,14 @@ def sanitize_module_name(module_name): return module_name +def extract_domain(url): + """Extract domain name from URL string""" + o = urlparse(url) + if o.scheme == '' and o.netloc == '': + o = urlparse("//" + url.lstrip("/")) + return o.netloc + + class Command(ScrapyCommand): requires_project = False @@ -59,7 +68,8 @@ class Command(ScrapyCommand): if len(args) != 2: raise UsageError() - name, domain = args[0:2] + name, url = args[0:2] + domain = extract_domain(url) module = sanitize_module_name(name) if self.settings.get('BOT_NAME') == module: diff --git a/tests/test_commands.py b/tests/test_commands.py index 74b917d93..086286b3a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import json import optparse import os import platform +import re import subprocess import sys import tempfile @@ -94,6 +95,15 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) + def find_in_file(self, filename, regex): + """Find first pattern occurrence in file""" + pattern = re.compile(regex) + with open(filename, "r") as f: + for line in f: + match = pattern.search(line) + if match is not None: + return match + class StartprojectTest(ProjectTest): @@ -482,6 +492,26 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) + def test_url(self, url='test.com', domain="test.com"): + self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) + self.assertEqual(domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) + self.assertEqual('http://%s/' % domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) + + def test_url_schema(self): + self.test_url('http://test.com', 'test.com') + + def test_url_path(self): + self.test_url('test.com/some/other/page', 'test.com') + + def test_url_schema_path(self): + self.test_url('https://test.com/some/other/page', 'test.com') + class GenspiderStandaloneCommandTest(ProjectTest): From 43ea21e8306bc66e8f07abc84cce680726abc7dc Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Tue, 24 Aug 2021 15:18:01 +0530 Subject: [PATCH 1787/1853] Feed post-processing plugin support (#5190) --- .github/workflows/tests-ubuntu.yml | 2 +- docs/topics/feed-exports.rst | 67 +++- scrapy/extensions/feedexport.py | 4 + scrapy/extensions/postprocessing.py | 154 +++++++++ tests/test_feedexport.py | 496 ++++++++++++++++++++++++++++ 5 files changed, 721 insertions(+), 2 deletions(-) create mode 100644 scrapy/extensions/postprocessing.py diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 81beda5da..ef1c8362f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -20,7 +20,7 @@ jobs: - python-version: pypy3 env: TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.1 + PYPY_VERSION: 3.6-v7.3.3 # pinned deps - python-version: 3.6.12 diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2b3217d62..116967280 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. + .. _item-filter: Item filtering @@ -312,6 +313,63 @@ ItemFilter :members: +.. _post-processing: + +Post-Processing +=============== + +.. versionadded:: VERSION + +Scrapy provides an option to activate plugins to post-process feeds before they are exported +to feed storages. In addition to using :ref:`builtin plugins `, you +can create your own :ref:`plugins `. + +These plugins can be activated through the ``postprocessing`` option of a feed. +The option must be passed a list of post-processing plugins in the order you want +the feed to be processed. These plugins can be declared either as an import string +or with the imported class of the plugin. Parameters to plugins can be passed +through the feed options. See :ref:`feed options ` for examples. + +.. _builtin-plugins: + +Built-in Plugins +---------------- + +.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin + +.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin + +.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin + +.. _custom-plugins: + +Custom Plugins +-------------- + +Each plugin is a class that must implement the following methods: + +.. method:: __init__(self, file, feed_options) + + Initialize the plugin. + + :param file: file-like object having at least the `write`, `tell` and `close` methods implemented + + :param feed_options: feed-specific :ref:`options ` + :type feed_options: :class:`dict` + +.. method:: write(self, data) + + Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file. + It must return number of bytes written. + +.. method:: close(self) + + Close the target file object. + +To pass a parameter to your plugin, use :ref:`feed options `. You +can then access those parameters from the ``__init__`` method of your plugin. + + Settings ======== @@ -368,10 +426,12 @@ For instance:: 'encoding': 'latin1', 'indent': 8, }, - pathlib.Path('items.csv'): { + pathlib.Path('items.csv.gz'): { 'format': 'csv', 'fields': ['price', 'name'], 'item_filter': 'myproject.filters.MyCustomFilter2', + 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 5, }, } @@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition: - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. +- ``postprocessing``: list of :ref:`plugins ` to use for post-processing. + + The plugins will be used in the order of the list passed. + + .. versionadded:: VERSION .. setting:: FEED_EXPORT_ENCODING diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 564c736f2..0f5bf01d0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -20,6 +20,7 @@ from zope.interface import implementer, Interface from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file @@ -396,6 +397,9 @@ class FeedExporter: """ storage = self._get_storage(uri, feed_options) file = storage.open(spider) + if "postprocessing" in feed_options: + file = PostProcessingManager(feed_options["postprocessing"], file, feed_options) + exporter = self._get_exporter( file=file, format=feed_options['format'], diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py new file mode 100644 index 000000000..413c2e55e --- /dev/null +++ b/scrapy/extensions/postprocessing.py @@ -0,0 +1,154 @@ +""" +Extension for processing data before they are exported to feeds. +""" +from bz2 import BZ2File +from gzip import GzipFile +from io import IOBase +from lzma import LZMAFile +from typing import Any, BinaryIO, Dict, List + +from scrapy.utils.misc import load_object + + +class GzipPlugin: + """ + Compresses received data using `gzip `_. + + Accepted ``feed_options`` parameters: + + - `gzip_compresslevel` + - `gzip_mtime` + - `gzip_filename` + + See :py:class:`gzip.GzipFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("gzip_compresslevel", 9) + mtime = self.feed_options.get("gzip_mtime") + filename = self.feed_options.get("gzip_filename") + self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level, + mtime=mtime, filename=filename) + + def write(self, data: bytes) -> int: + return self.gzipfile.write(data) + + def close(self) -> None: + self.gzipfile.close() + self.file.close() + + +class Bz2Plugin: + """ + Compresses received data using `bz2 `_. + + Accepted ``feed_options`` parameters: + + - `bz2_compresslevel` + + See :py:class:`bz2.BZ2File` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("bz2_compresslevel", 9) + self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level) + + def write(self, data: bytes) -> int: + return self.bz2file.write(data) + + def close(self) -> None: + self.bz2file.close() + self.file.close() + + +class LZMAPlugin: + """ + Compresses received data using `lzma `_. + + Accepted ``feed_options`` parameters: + + - `lzma_format` + - `lzma_check` + - `lzma_preset` + - `lzma_filters` + + .. note:: + ``lzma_filters`` cannot be used in pypy version 7.3.1 and older. + + See :py:class:`lzma.LZMAFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + + format = self.feed_options.get("lzma_format") + check = self.feed_options.get("lzma_check", -1) + preset = self.feed_options.get("lzma_preset") + filters = self.feed_options.get("lzma_filters") + self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format, + check=check, preset=preset, filters=filters) + + def write(self, data: bytes) -> int: + return self.lzmafile.write(data) + + def close(self) -> None: + self.lzmafile.close() + self.file.close() + + +# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager +# instance as a file like writable object. This could be needed by some exporters +# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper. +class PostProcessingManager(IOBase): + """ + This will manage and use declared plugins to process data in a + pipeline-ish way. + :param plugins: all the declared plugins for the feed + :type plugins: list + :param file: final target file where the processed data will be written + :type file: file like object + """ + + def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.plugins = self._load_plugins(plugins) + self.file = file + self.feed_options = feed_options + self.head_plugin = self._get_head_plugin() + + def write(self, data: bytes) -> int: + """ + Uses all the declared plugins to process data first, then writes + the processed data to target file. + :param data: data passed to be written to target file + :type data: bytes + :return: returns number of bytes written + :rtype: int + """ + return self.head_plugin.write(data) + + def tell(self) -> int: + return self.file.tell() + + def close(self) -> None: + """ + Close the target file along with all the plugins. + """ + self.head_plugin.close() + + def writable(self) -> bool: + return True + + def _load_plugins(self, plugins: List[Any]) -> List[Any]: + plugins = [load_object(plugin) for plugin in plugins] + return plugins + + def _get_head_plugin(self) -> Any: + prev = self.file + for plugin in self.plugins[::-1]: + prev = plugin(prev, self.feed_options) + return prev diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 53e6a2018..253f3119c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,5 +1,8 @@ +import bz2 import csv +import gzip import json +import lzma import os import random import shutil @@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(row['expected'], data[feed_options['format']]) +class FeedPostProcessedExportsTest(FeedExportTestBase): + __test__ = True + + items = [{'foo': 'bar'}] + expected = b'foo\r\nbar\r\n' + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get('plugin1_char', b'') + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name): + return os.path.join(self.temp_dir, name) + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data with filename. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content = {} + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for file_path, feed_options in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + + with open(str(file_path), 'rb') as f: + content[str(file_path)] = f.read() + + finally: + for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + + os.remove(str(file_path)) + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''): + data_stream = BytesIO() + gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime, + compresslevel=compresslevel, mode="wb") + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @defer.inlineCallbacks + def test_gzip_plugin(self): + + filename = self._named_tempfile('gzip_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + self.fail("Received invalid gzip data.") + + @defer.inlineCallbacks + def test_gzip_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0), + self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 0, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 9, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123), + self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789), + } + + settings = { + 'FEEDS': { + self._named_tempfile('mtime_123'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123, + 'gzip_filename': "", + }, + self._named_tempfile('mtime_123456789'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123456789, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"), + self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"), + } + + settings = { + 'FEEDS': { + self._named_tempfile('filename_FILE1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE1", + }, + self._named_tempfile('filename_FILE2'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE2", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin(self): + + filename = self._named_tempfile('lzma_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + self.fail("Received invalid lzma data.") + + @defer.inlineCallbacks + def test_lzma_plugin_format(self): + + filename_to_compressed = { + self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ), + self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE), + } + + settings = { + 'FEEDS': { + self._named_tempfile('format_FORMAT_XZ'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_XZ, + }, + self._named_tempfile('format_FORMAT_ALONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_ALONE, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_check(self): + + filename_to_compressed = { + self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE), + self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256), + } + + settings = { + 'FEEDS': { + self._named_tempfile('check_CHECK_NONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_NONE, + }, + self._named_tempfile('check_CHECK_CRC256'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_SHA256, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_preset(self): + + filename_to_compressed = { + self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0), + self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('preset_PRESET_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 0, + }, + self._named_tempfile('preset_PRESET_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_filters(self): + import sys + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + raise unittest.SkipTest("lzma filters doesn't work in PyPy") + + filters = [{'id': lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile('filters') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_filters': filters, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(compressed, data[filename]) + result = lzma.decompress(data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_bz2_plugin(self): + + filename = self._named_tempfile('bz2_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + self.fail("Received invalid bz2 data.") + + @defer.inlineCallbacks + def test_bz2_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1), + self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 1, + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_custom_plugin(self): + filename = self._named_tempfile('csv_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(self.expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_parameter(self): + + expected = b'foo\r\n\nbar\r\n\n' + filename = self._named_tempfile('newline') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + 'plugin1_char': b'\n' + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_compression(self): + + expected = b'foo\r\n\nbar\r\n\n' + + filename_to_decompressor = { + self._named_tempfile('bz2'): bz2.decompress, + self._named_tempfile('lzma'): lzma.decompress, + self._named_tempfile('gzip'): gzip.decompress, + } + + settings = { + 'FEEDS': { + self._named_tempfile('bz2'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('lzma'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('gzip'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'plugin1_char': b'\n', + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def test_exports_compatibility_with_postproc(self): + import marshal + import pickle + filename_to_expected = { + self._named_tempfile('csv'): b'foo\r\nbar\r\n', + self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]', + self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n', + self._named_tempfile('xml'): b'\n' + b'\nbar\n', + } + + settings = { + 'FEEDS': { + self._named_tempfile('csv'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile('json'): { + 'format': 'json', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('jsonlines'): { + 'format': 'jsonlines', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('xml'): { + 'format': 'xml', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('marshal'): { + 'format': 'marshal', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('pickle'): { + 'format': 'pickle', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, result in data.items(): + if 'pickle' in filename: + expected, result = self.items[0], pickle.loads(result) + elif 'marshal' in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + self.assertEqual(expected, result) + + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)02d_' From 8284de5e7613c47e69d40d4f6a2a1dc846b50dd6 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 24 Aug 2021 15:15:29 +0500 Subject: [PATCH 1788/1853] Fix/silence the Pylint messages added in 2.10 (#5235) --- pylintrc | 1 + scrapy/utils/conf.py | 2 +- scrapy/utils/deprecate.py | 2 +- tests/keys/__init__.py | 8 ++++---- tests/test_downloader_handlers_http2.py | 2 +- tests/test_engine.py | 2 +- tests/test_exporters.py | 4 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_request_cb_kwargs.py | 2 +- tests/test_scheduler.py | 10 +++++----- tests/test_utils_conf.py | 4 ++-- tests/test_utils_misc/__init__.py | 2 +- 13 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pylintrc b/pylintrc index a44712507..699686e16 100644 --- a/pylintrc +++ b/pylintrc @@ -105,6 +105,7 @@ disable=abstract-method, unnecessary-lambda, unnecessary-pass, unreachable, + unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index b904c4a03..24873f75d 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) - out.setdefault("item_export_kwargs", dict()) + out.setdefault("item_export_kwargs", {}) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f5b17416f..ae727464c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -79,7 +79,7 @@ def create_deprecated_class( # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) - for c in {type(inst), inst.__class__}) + for c in (type(inst), inst.__class__)) def __subclasscheck__(cls, sub): if cls is not DeprecatedClass.deprecated_class: diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index da202be4d..bb4a8e5af 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -40,9 +40,9 @@ def generate_keys(): subject = issuer = Name( [ - NameAttribute(NameOID.COUNTRY_NAME, u"IE"), - NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), - NameAttribute(NameOID.COMMON_NAME, u"localhost"), + NameAttribute(NameOID.COUNTRY_NAME, "IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"), + NameAttribute(NameOID.COMMON_NAME, "localhost"), ] ) cert = ( @@ -54,7 +54,7 @@ def generate_keys(): .not_valid_before(datetime.utcnow()) .not_valid_after(datetime.utcnow() + timedelta(days=10)) .add_extension( - SubjectAlternativeName([DNSName(u"localhost")]), + SubjectAlternativeName([DNSName("localhost")]), critical=False, ) .sign(key, SHA256(), default_backend()) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 53bb4fe92..8c8c30597 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -219,7 +219,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): certfile = 'keys/localhost.crt' scheme = 'https' - host = u'127.0.0.1' + host = '127.0.0.1' expected_http_proxy_request_body = b'/' diff --git a/tests/test_engine.py b/tests/test_engine.py index 92bf45f25..fa7d0c8d4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -152,7 +152,7 @@ class CrawlerRun: self.itemerror = [] self.itemresp = [] self.headers = {} - self.bytes = defaultdict(lambda: list()) + self.bytes = defaultdict(list) self.signals_caught = {} self.spider_class = spider_class diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 04bae31d3..b263b3475 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -362,14 +362,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), expected=None, encoding='windows-1251', ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), include_headers_line=False, expected='Wɵ​rd\r\n', encoding='windows-1251', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 677ede92b..49c83132f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -201,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase): self.site = Site(root, timeout=None) # Start server for testing - self.hostname = u'localhost' + self.hostname = 'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 41afa2896..0fd52da5f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): return None with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context=dict()) + wrap_loader_context(function, context={}) assert len(w) == 1 assert issubclass(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b68184b87..738502de8 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -57,7 +57,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = list() + checks = [] def start_requests(self): data = {'key': 'value', 'number': 123} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 512a7460e..2d4bfa165 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active']) class MockDownloader: def __init__(self): - self.slots = dict() + self.slots = {} def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: @@ -31,7 +31,7 @@ class MockDownloader: return urlparse_cached(request).hostname or '' def increment(self, slot_key): - slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) def decrement(self, slot_key): @@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): for url, priority in _PRIORITIES: self.scheduler.enqueue_request(Request(url, priority=priority)) - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin: self.close_scheduler() self.create_scheduler() - dequeued_slots = list() + dequeued_slots = [] requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc2560add..a92880626 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 47d73a2dd..b83c1d6f0 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - self.assertRaises(TypeError, load_object, dict()) + self.assertRaises(TypeError, load_object, {}) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') From ac9175964dda07da1e838ebb063fc8dd95925e0d Mon Sep 17 00:00:00 2001 From: maanijou <19888963+maanijou@users.noreply.github.com> Date: Sun, 12 Sep 2021 17:59:20 +0200 Subject: [PATCH 1789/1853] Improve documentation for spider middlewares --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f0158dc41..f3fb0d5d7 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects and :ref:`item object + iterable of :class:`~scrapy.Request` objects or :ref:`item object `. If it returns ``None``, Scrapy will continue processing this exception, From e5998fb8469dff1e2e965826d2b43f9bad0c17ad Mon Sep 17 00:00:00 2001 From: kamran890 Date: Wed, 22 Sep 2021 03:00:18 +0500 Subject: [PATCH 1790/1853] Document spider.state attribute (#5174) --- docs/topics/jobs.rst | 2 ++ docs/topics/spiders.rst | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index e49f37a2f..f16d306c7 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 +.. _topics-keeping-persistent-state-between-batches: + Keeping persistent state between batches ======================================== diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 67b9e2e0e..4d3d32941 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -122,6 +122,11 @@ scrapy.Spider send log messages through it as described on :ref:`topics-logging-from-spiders`. + .. attribute:: state + + A dict you can use to persist some spider state between batches. + See :ref:`topics-keeping-persistent-state-between-batches` to know more about it. + .. method:: from_crawler(crawler, *args, **kwargs) This is the class method used by Scrapy to create your spiders. From 1829dd774ca0f056e97f7c4621575ef9126e4b57 Mon Sep 17 00:00:00 2001 From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com> Date: Sat, 25 Sep 2021 20:22:09 +0330 Subject: [PATCH 1791/1853] Update spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f3fb0d5d7..08be2b03c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects or :ref:`item object + iterable of :class:`~scrapy.Request` objects or :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From dfdb779756aa1df2059980de02d359e4e93bac55 Mon Sep 17 00:00:00 2001 From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com> Date: Sun, 26 Sep 2021 12:45:44 +0330 Subject: [PATCH 1792/1853] Apply review comments --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 08be2b03c..f1373b9ee 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects or :ref:`item objects + iterable of :class:`~scrapy.Request` or :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From 3c57825b0f3a7bb250aec753b09f964f36500e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 26 Sep 2021 13:41:26 +0200 Subject: [PATCH 1793/1853] Update docs/topics/spider-middleware.rst --- docs/topics/spider-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f1373b9ee..73bedf655 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` or :ref:`item objects - `. + iterable of :class:`~scrapy.Request` or :ref:`item ` + objects. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_spider_exception` in the following From 74f146bbe0c41e2c21f431c00ff34d6c5d10cb63 Mon Sep 17 00:00:00 2001 From: Deepanshu <73387559+iDeepverma@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:47:05 +0530 Subject: [PATCH 1794/1853] Document update URLLENGTH_LIMIT --- docs/topics/settings.rst | 8 ++++++-- docs/topics/spider-middleware.rst | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2ab2020fa..4d3ae20cc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1645,8 +1645,12 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` -The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +The maximum URL length to allow for crawled URLs. You can set this to ``0`` +to disable :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` for working +with URLs longer than the default value. The default limit acts as a stopping condition in case of +URLs of increasing length, usually caused by a loop. +For more information about the default value +for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 73bedf655..a0a7b1fb6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,4 +440,5 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + If ``0``, then :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is disabled. From 890f884de46602352de48fa844edd9959c62e473 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 1 Oct 2021 04:50:42 -0300 Subject: [PATCH 1795/1853] Allow 'callback' key in keyword arguments for request callbacks (#5251) --- scrapy/core/scraper.py | 2 +- tests/test_request_cb_kwargs.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d6d6f64f9..f40bccbb3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -156,7 +156,7 @@ class Scraper: callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) - dfd.addCallback(callback, **result.request.cb_kwargs) + dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 738502de8..8b96fe1a1 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -60,7 +60,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks = [] def start_requests(self): - data = {'key': 'value', 'number': 123} + data = {'key': 'value', 'number': 123, 'callback': 'some_callback'} yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) @@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider): if response.url.endswith('/general_with'): self.checks.append(kwargs['key'] == 'value') self.checks.append(kwargs['number'] == 123) - self.crawler.stats.inc_value('boolean_checks', 2) + self.checks.append(kwargs['callback'] == 'some_callback') + self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): self.checks.append(kwargs == {}) self.crawler.stats.inc_value('boolean_checks') @@ -110,7 +111,7 @@ class KeywordArgumentsSpider(MockServerSpider): TypeError: parse_takes_less() got an unexpected keyword argument 'number' """ - def parse_takes_more(self, response, key, number, other): + def parse_takes_more(self, response, key, number, callback, other): """ Should raise TypeError: parse_takes_more() missing 1 required positional argument: 'other' @@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase): self.assertTrue( str(exceptions['takes_less'].exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" - ) + ), + msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]), ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) self.assertTrue( str(exceptions['takes_more'].exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" - ) + ), + msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]), ) From fbb1236fd6ee283414360f5209b3a569e112c8cb Mon Sep 17 00:00:00 2001 From: Deepanshu verma <73387559+iDeepverma@users.noreply.github.com> Date: Fri, 1 Oct 2021 18:46:11 +0530 Subject: [PATCH 1796/1853] Update docs/topics/settings.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit added suggestion Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4d3ae20cc..ed8f1f105 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1645,12 +1645,19 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` -The maximum URL length to allow for crawled URLs. You can set this to ``0`` -to disable :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` for working -with URLs longer than the default value. The default limit acts as a stopping condition in case of -URLs of increasing length, usually caused by a loop. -For more information about the default value -for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +The maximum URL length to allow for crawled URLs. + +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +:setting:`DEPTH_LIMIT`. + +Use ``0`` to allow URLs of any length. + +The default value is copied from the `Microsoft Internet Explorer maximum URL +length`_, even though this setting exists for different reasons. + +.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT From d91d82b5064c512e741da106b5fb3a398027d5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Oct 2021 16:31:29 +0200 Subject: [PATCH 1797/1853] Make Scrapy SFW again --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d3e08efd4..42ce22158 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -300,7 +300,7 @@ errors if needed:: "http://www.httpbin.org/status/404", # Not found error "http://www.httpbin.org/status/500", # server issue "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "http://www.httphttpbinbin.org/", # DNS error expected + "https://example.invalid/", # DNS error expected ] def start_requests(self): From de2043f9c1661208de7b73305a8e3a2395d29583 Mon Sep 17 00:00:00 2001 From: Deepanshu <73387559+iDeepverma@users.noreply.github.com> Date: Fri, 1 Oct 2021 20:20:00 +0530 Subject: [PATCH 1798/1853] updated docs/topics/spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a0a7b1fb6..f27bc79c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,5 +440,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - If ``0``, then :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is disabled. - From 47533985f4dc7e58895e1a34c3ea88502f83572a Mon Sep 17 00:00:00 2001 From: Peter Morrison Date: Fri, 1 Oct 2021 12:30:14 -0600 Subject: [PATCH 1799/1853] Document file expiration method in media-pipeline --- docs/topics/media-pipeline.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 46bd2859b..10d2ac990 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -383,6 +383,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. +The last modified time from the file is used to determine the age of the file in days, +which is then compared to the set expiration time to determine if the file is expired. + .. _topics-images-thumbnails: Thumbnail generation for images From de70b3c58b5b4b2f95468218c194b1e4b99a33c4 Mon Sep 17 00:00:00 2001 From: Deepanshu verma <73387559+iDeepverma@users.noreply.github.com> Date: Sat, 2 Oct 2021 12:12:58 +0530 Subject: [PATCH 1800/1853] Update spider-middleware.rst added empty line --- docs/topics/spider-middleware.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f27bc79c0..3545e760b 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,3 +440,4 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + From f10880022273c005a6cb6d9e6ff9cc9a370cc375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 2 Oct 2021 13:25:15 +0200 Subject: [PATCH 1801/1853] Update spider-middleware.rst --- docs/topics/spider-middleware.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 3545e760b..f27bc79c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,4 +440,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - From ef263042d75586e94d74290d1a41c432f7d87bec Mon Sep 17 00:00:00 2001 From: Raihan Nismara <31585789+raihan71@users.noreply.github.com> Date: Sun, 3 Oct 2021 13:26:20 +0700 Subject: [PATCH 1802/1853] Using Logo Scrapy in Readme.md Logo scrapy used in readme.md made looks nicer --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 5750e2c0f..05f10bb6c 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,6 @@ +.. image:: /artwork/scrapy-logo.jpg + :width: 400px + ====== Scrapy ====== From b9647b85d3bc9dcefc7d829ce8304bb28f7cc798 Mon Sep 17 00:00:00 2001 From: Ryan Whelchel Date: Sun, 3 Oct 2021 17:32:38 -0400 Subject: [PATCH 1803/1853] docs: restructed phrasing for clarity --- docs/intro/tutorial.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 438f3d6df..ba27b18bc 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -277,9 +277,9 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` -instance avoids an ``IndexError`` and returns ``None`` when it doesn't -find any element matching the selection. +Directly accessing an index on a :class:`~scrapy.selector.SelectorList` +instance could potentially run into an ``IndexError``. It is recommended to use +``.get()`` directly instead as it avoids such index errors. There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail From 764cf0178bb7bc346f1c15ed7ab0adcbb75c43b0 Mon Sep 17 00:00:00 2001 From: Ryan Whelchel Date: Tue, 5 Oct 2021 10:22:57 -0400 Subject: [PATCH 1804/1853] Update docs/intro/tutorial.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/intro/tutorial.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ba27b18bc..fa321a770 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -277,9 +277,20 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -Directly accessing an index on a :class:`~scrapy.selector.SelectorList` -instance could potentially run into an ``IndexError``. It is recommended to use -``.get()`` directly instead as it avoids such index errors. +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +raise an :exc:`IndexError` exception if there are no results:: + + >>> response.css('noelement')[0].get() + Traceback (most recent call last): + File "", line 1, in + ... + IndexError: list index out of range + +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +if there are no results:: + +>>> response.css("noelement").get() There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail From b081f18a2f232a1bfd04c829ae6894cac93a89a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 16 Aug 2019 14:53:42 +0500 Subject: [PATCH 1805/1853] Add http_auth_domain to HttpAuthMiddleware. --- docs/topics/downloader-middleware.rst | 18 ++++- scrapy/downloadermiddlewares/httpauth.py | 21 ++++- tests/test_downloadermiddleware_httpauth.py | 87 ++++++++++++++++++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 99d57bda9..28b019c80 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -323,8 +323,21 @@ HttpAuthMiddleware This middleware authenticates all requests generated from certain spiders using `Basic access authentication`_ (aka. HTTP auth). - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication for a spider, set the ``http_user`` and + ``http_pass`` spider attributes to the authentication data and the + ``http_auth_domain`` spider attribute to the domain which requires this + authentication (its subdomains will be also handled in the same way). + You can set ``http_auth_domain`` to ``None`` to enable the + authentication for all requests but usually this is not needed. + + .. warning:: + In the previous Scrapy versions HttpAuthMiddleware sent the + authentication data with all requests, which is a security problem if + the spider makes requests to several different domains. Currently if + the ``http_auth_domain`` attribute is not set, the middleware will use + the domain of the first request, which will work for some spider but + not for others. In the future the middleware will produce an error + instead. Example:: @@ -334,6 +347,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' + http_auth_domain = 'intranet.example.com' name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 089bf0d85..1bee3e279 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,10 +3,14 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +import warnings from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.url import url_is_from_any_domain class HttpAuthMiddleware: @@ -24,8 +28,23 @@ class HttpAuthMiddleware: pwd = getattr(spider, 'http_pass', '') if usr or pwd: self.auth = basic_auth_header(usr, pwd) + if not hasattr(spider, 'http_auth_domain'): + warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security ' + 'problems if the spider makes requests to several different domains. http_auth_domain ' + 'will be set to the domain of the first request, please set it to the correct value ' + 'explicitly.', + category=ScrapyDeprecationWarning) + self.domain_unset = True + else: + self.domain = spider.http_auth_domain + self.domain_unset = False def process_request(self, request, spider): auth = getattr(self, 'auth', None) if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth + domain = urlparse_cached(request).hostname + if self.domain_unset: + self.domain = domain + self.domain_unset = False + if not self.domain or url_is_from_any_domain(request.url, [self.domain]): + request.headers[b'Authorization'] = auth diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 3381632b0..0362e2018 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,13 +1,60 @@ import unittest +from w3lib.http import basic_auth_header + from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider +class TestSpiderLegacy(Spider): + http_user = 'foo' + http_pass = 'bar' + + class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' + http_auth_domain = 'example.com' + + +class TestSpiderAny(Spider): + http_user = 'foo' + http_pass = 'bar' + http_auth_domain = None + + +class HttpAuthMiddlewareLegacyTest(unittest.TestCase): + + def setUp(self): + self.spider = TestSpiderLegacy('foo') + + def test_auth(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + + # initial request, sets the domain and sends the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to the same domain, should send the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to a different domain, shouldn't send the header + req = Request('http://example-noauth.com/') + assert mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_auth_already_set(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') class HttpAuthMiddlewareTest(unittest.TestCase): @@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw - def test_auth(self): - req = Request('http://scrapytest.org/') + def test_no_auth(self): + req = Request('http://example-noauth.com/') assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertNotIn('Authorization', req.headers) + + def test_auth_domain(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_subdomain(self): + req = Request('http://foo.example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) def test_auth_already_set(self): - req = Request('http://scrapytest.org/', + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') + + +class HttpAuthAnyMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpiderAny('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_already_set(self): + req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers['Authorization'], b'Digest 123') From 7ec5f299c42d07768c02970f0a11f018ed790188 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 22 Aug 2019 20:32:56 +0500 Subject: [PATCH 1806/1853] Small documentation fixes. --- docs/topics/downloader-middleware.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 28b019c80..caf44a903 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -328,16 +328,16 @@ HttpAuthMiddleware ``http_auth_domain`` spider attribute to the domain which requires this authentication (its subdomains will be also handled in the same way). You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but usually this is not needed. + authentication for all requests but you risk leaking your authentication + credentials to unrelated domains. .. warning:: - In the previous Scrapy versions HttpAuthMiddleware sent the - authentication data with all requests, which is a security problem if - the spider makes requests to several different domains. Currently if - the ``http_auth_domain`` attribute is not set, the middleware will use - the domain of the first request, which will work for some spider but - not for others. In the future the middleware will produce an error - instead. + In previous Scrapy versions HttpAuthMiddleware sent the authentication + data with all requests, which is a security problem if the spider + makes requests to several different domains. Currently if the + ``http_auth_domain`` attribute is not set, the middleware will use the + domain of the first request, which will work for some spiders but not + for others. In the future the middleware will produce an error instead. Example:: From f0105a882df200f71088603fecaeb9d40679c387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Oct 2021 13:29:06 +0200 Subject: [PATCH 1807/1853] Cover 2.5.1 in the release notes --- docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0ea412e75..4b5cbb2da 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,44 @@ Release notes ============= +.. _release-2.5.1: + +Scrapy 2.5.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-2.5.0: Scrapy 2.5.0 (2021-04-06) From 735750c254e6e82af46b4ebbb35e28b8c0a52250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Oct 2021 21:10:49 +0200 Subject: [PATCH 1808/1853] Cover 1.8.1 in the release notes --- docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 4b5cbb2da..5e590f027 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1492,6 +1492,44 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.1: + +Scrapy 1.8.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-1.8.0: Scrapy 1.8.0 (2019-10-28) From 6c858cec91b013853a73e6215b74c90b609bc2da Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Oct 2021 12:32:04 -0300 Subject: [PATCH 1809/1853] Cookies: Cast primitive types to str (#5253) * cast primitive types to str * add tests --- scrapy/downloadermiddlewares/cookies.py | 4 ++-- tests/test_downloadermiddleware_cookies.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d95ed3d38..0eee8d758 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -80,8 +80,8 @@ class CookiesMiddleware: logger.warning(msg.format(request, cookie, key)) return continue - if isinstance(cookie[key], str): - decoded[key] = cookie[key] + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) else: try: decoded[key] = cookie[key].decode("utf8") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index aff8542e9..36021bfbf 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -347,3 +347,24 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') self.assertCookieValEqual(req3.headers['Cookie'], 'key=') + + def test_primitive_type_cookies(self): + # Boolean + req1 = Request('http://example.org', cookies={'a': True}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=True') + + # Float + req2 = Request('http://example.org', cookies={'a': 9.5}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5') + + # Integer + req3 = Request('http://example.org', cookies={'a': 10}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=10') + + # String + req4 = Request('http://example.org', cookies={'a': 'b'}) + assert self.mw.process_request(req4, self.spider) is None + self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') From b1cb007b3b8fef6f037cd1abd38fe9da7190ed26 Mon Sep 17 00:00:00 2001 From: MarvinPetzoldt <78762153+MarvinPetzoldt@users.noreply.github.com> Date: Wed, 6 Oct 2021 19:08:19 +0200 Subject: [PATCH 1810/1853] Fixed documentation example --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8c30122b6..923336769 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -122,7 +122,7 @@ Example:: class ProductXmlExporter(XmlItemExporter): def serialize_field(self, field, name, value): - if field == 'price': + if name == 'price': return f'$ {str(value)}' return super().serialize_field(field, name, value) From 029cab72e8a9b20ebb6b9540e9e01747f0e83dba Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 6 Oct 2021 14:34:09 -0300 Subject: [PATCH 1811/1853] [CI] fix pypy test (#5264) --- tests/test_request_cb_kwargs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8b96fe1a1..473a93e69 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -105,7 +105,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(default == 99) self.crawler.stats.inc_value('boolean_checks', 4) - def parse_takes_less(self, response, key): + def parse_takes_less(self, response, key, callback): """ Should raise TypeError: parse_takes_less() got an unexpected keyword argument 'number' From d3f1bf79e883fe3662df827ee47cfc93a372ff02 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 7 Oct 2021 17:27:20 +0300 Subject: [PATCH 1812/1853] Use f-strings where appropriate (#5246) --- scrapy/__init__.py | 2 +- scrapy/core/downloader/contextfactory.py | 12 +++++---- scrapy/core/downloader/tls.py | 8 +++--- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/extensions/feedexport.py | 33 ++++++++++------------- scrapy/extensions/httpcache.py | 4 +-- scrapy/utils/misc.py | 2 +- tests/spiders.py | 16 +++++------ tests/test_closespider.py | 2 +- tests/test_commands.py | 4 +-- tests/test_downloader_handlers_http2.py | 2 +- tests/test_http_request.py | 19 +++++++------ 12 files changed, 51 insertions(+), 55 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 8a8065bf2..396f98219 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version if sys.version_info < (3, 6): - print("Scrapy %s requires Python 3.6+" % __version__) + print(f"Scrapy {__version__} requires Python 3.6+") sys.exit(1) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 073ef16bf..b5318c7bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler): settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = ( + f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " + "a `method` argument (type OpenSSL.SSL method, e.g. " + "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " + "argument and/or a `tls_ciphers` argument. Please, upgrade your " + "context factory class to handle them or ignore them." + ) warnings.warn(msg) return context_factory diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 2b8990b75..19a56d9b6 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions): verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( - 'Remote certificate is not valid for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + 'Remote certificate is not valid for hostname "%s"; %s', + self._hostnameASCII, e) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' - 'from host "{}" (exception: {})'.format( - self._hostnameASCII, repr(e))) + 'from host "%s" (exception: %r)', + self._hostnameASCII, e) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 62f1c3a29..80ed7ac75 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -70,7 +70,7 @@ class HttpCacheMiddleware: self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) - raise IgnoreRequest("Ignored request not in cache: %s" % request) + raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0f5bf01d0..370723368 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): kwargs['feed_options'] = feed_options else: warnings.warn( - "{} does not support the 'feed_options' keyword argument. Add a " + f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " "'feed_options' parameter to its signature to remove this " "warning. This parameter will become mandatory in a future " - "version of Scrapy." - .format(builder.__qualname__), + "version of Scrapy.", category=ScrapyDeprecationWarning ) return builder(*preargs, uri, *args, **kwargs) @@ -356,32 +355,28 @@ class FeedExporter: # properly closed. return defer.maybeDeferred(slot.storage.store, slot.file) slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) - # Use `largs=log_args` to copy log_args into function's scope - # instead of using `log_args` from the outer scope d.addCallback( - self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_success, logmsg, spider, type(slot.storage).__name__ ) d.addErrback( - self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) return d - def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + def _handle_store_error(self, f, logmsg, spider, slot_type): logger.error( - logfmt % "Error storing", largs, + "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + def _handle_store_success(self, f, logmsg, spider, slot_type): logger.info( - logfmt % "Stored", largs, extra={'spider': spider} + "Stored %s", logmsg, + extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") @@ -474,10 +469,10 @@ class FeedExporter: for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ' 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' - 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' - ''.format(uri_template) + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count', + uri_template ) return False return True @@ -526,7 +521,7 @@ class FeedExporter: instance = build_instance(feedcls) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance def _get_uri_params(self, spider, uri_params, slot=None): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index e0c04b2de..d0ae29b90 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -226,7 +226,7 @@ class DbmCacheStorage: dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') - logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) def close_spider(self, spider): self.db.close() @@ -280,7 +280,7 @@ class FilesystemCacheStorage: self._open = gzip.open if self.use_gzip else open def open_spider(self, spider): - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) def close_spider(self, spider): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 51cef1e91..11c4206c2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -50,7 +50,7 @@ def load_object(path): return path else: raise TypeError("Unexpected argument type, expected string " - "or object, got: %s" % type(path)) + f"or object, got: {type(path)}") try: dot = path.rindex('.') diff --git a/tests/spiders.py b/tests/spiders.py index 5b45f897e..67dbbbe0f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider): self.start_urls = [url] def parse(self, response): - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): @@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider): async def parse(self, response): await defer.succeed(42) - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioSpider(SimpleSpider): @@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") class AsyncDefAsyncioReturnSpider(SimpleSpider): @@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return [{'id': 1}, {'id': 2}] @@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return {"foo": 42} @@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): await asyncio.sleep(0.2) req_id = response.meta.get('req_id', 0) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d, req_id %d" % (status, req_id)) + self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: return reqs = [] @@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) yield {'foo': 42} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenLoopSpider(SimpleSpider): @@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): for i in range(10): await asyncio.sleep(0.1) yield {'foo': i} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenComplexSpider(SimpleSpider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 5ec5e2989..be8adadb3 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,7 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) + key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}' errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_commands.py b/tests/test_commands.py index 086286b3a..75098a77a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -498,7 +498,7 @@ class GenspiderCommandTest(CommandTest): self.find_in_file(join(self.proj_mod_path, 'spiders', 'test_name.py'), r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) - self.assertEqual('http://%s/' % domain, + self.assertEqual(f'http://{domain}/', self.find_in_file(join(self.proj_mod_path, 'spiders', 'test_name.py'), r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) @@ -708,7 +708,7 @@ class MySpider(scrapy.Spider): ]) import asyncio loop = asyncio.new_event_loop() - self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): spider_code = """ diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8c8c30597..3a9db3ee5 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'/') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f"{self.getURL('')}?noconnect" request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex( Warning, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index b610087bd..579ef9fa2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest): response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): - body = """ - - """ + form_methods = [[method, method] for method in self.request_class.valid_form_methods] + form_methods.append(['UNKNOWN', 'GET']) - for method in self.request_class.valid_form_methods: - response = _buildresponse(body % method) + for method, expected in form_methods: + response = _buildresponse( + f'
                ' + '' + '
                ' + ) r = self.request_class.from_response(response) - self.assertEqual(r.method, method) - - response = _buildresponse(body % 'UNKNOWN') - r = self.request_class.from_response(response) - self.assertEqual(r.method, 'GET') + self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): From 65d60b9692dc3475b42d14c744d8a5ac0f2b38cf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sun, 10 Oct 2021 05:06:36 -0300 Subject: [PATCH 1813/1853] [docs] add missing parameter to headers_received signal (#5270) --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a67cc1879..63ad3a9ad 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -413,7 +413,7 @@ headers_received .. versionadded:: 2.5 .. signal:: headers_received -.. function:: headers_received(headers, request, spider) +.. function:: headers_received(headers, body_length, request, spider) Sent by the HTTP 1.1 and S3 download handlers when the response headers are available for a given request, before downloading any additional content. From 6fbd6f941f00037ae4718805352e0fa86a781e41 Mon Sep 17 00:00:00 2001 From: ankur19 Date: Sat, 9 Oct 2021 19:09:51 -0400 Subject: [PATCH 1814/1853] Fix issue#5145 Fix condition for failing tests set Selector to None on AttributeError Add test and remove unused imports Fix imports --- scrapy/loader/__init__.py | 5 ++++- tests/test_loader.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 014951a8e..91337b949 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader): def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: - selector = self.default_selector_class(response) + try: + selector = self.default_selector_class(response) + except AttributeError: + selector = None context.update(response=response) super().__init__(item=item, selector=selector, parent=parent, **context) diff --git a/tests/test_loader.py b/tests/test_loader.py index b0bc82f4e..f7ab1f236 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,7 +4,7 @@ import attr from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response from scrapy.item import Item, Field from scrapy.loader import ItemLoader from scrapy.selector import Selector @@ -304,6 +304,12 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) + + def test_init_method_with_base_response(self): + """Selector should be None after initialization""" + response = Response("https://scrapy.org") + l = TestItemLoader(response=response) + self.assertIs(l.selector, None) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) From 3a263280bad53a26490381f293a454be6c25ea30 Mon Sep 17 00:00:00 2001 From: "Kian-Meng, Ang" Date: Mon, 11 Oct 2021 22:32:42 +0800 Subject: [PATCH 1815/1853] Fix typos --- docs/news.rst | 10 +++++----- docs/topics/settings.rst | 4 ++-- docs/topics/shell.rst | 2 +- docs/topics/spiders.rst | 4 ++-- docs/versioning.rst | 2 +- extras/qpsclient.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/exporters.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/files.py | 8 ++++---- scrapy/spiders/feed.py | 4 ++-- scrapy/utils/console.py | 2 +- scrapy/utils/datatypes.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/request.py | 4 ++-- sep/sep-001.rst | 2 +- sep/sep-005.rst | 2 +- sep/sep-014.rst | 2 +- sep/sep-021.rst | 2 +- tests/test_http_response.py | 4 ++-- tests/test_request_attribute_binding.py | 8 ++++---- tests/test_utils_defer.py | 4 ++-- tests/test_utils_template.py | 2 +- 23 files changed, 39 insertions(+), 39 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 5e590f027..509366c17 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1830,7 +1830,7 @@ New features * A new scheduler priority queue, ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant - scheduling improvement on crawls targetting multiple web domains, at the + scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) * A new :attr:`Request.cb_kwargs ` attribute @@ -2868,7 +2868,7 @@ Bug fixes - Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse ` (:issue:`2225`). - Fix for invalid JSON and XML files when spider yields no items (:issue:`872`). -- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). +- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). Refactoring ~~~~~~~~~~~ @@ -3731,7 +3731,7 @@ Scrapy 0.24.3 (2014-08-09) - adding some xpath tips to selectors docs (:commit:`2d103e0`) - fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`) - get_func_args maximum recursion fix #728 (:commit:`81344ea`) -- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`) +- Updated input/output processor example according to #560. (:commit:`f7c4ea8`) - Fixed Python syntax in tutorial. (:commit:`db59ed9`) - Add test case for tunneling proxy (:commit:`f090260`) - Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`) @@ -4393,7 +4393,7 @@ Scrapyd changes ~~~~~~~~~~~~~~~ - Scrapyd now uses one process per spider -- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) +- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default - There is now a ``scrapy server`` command to start a Scrapyd server of the current project @@ -4429,7 +4429,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- Split Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added ``dont_redirect`` request.meta key for avoiding redirects (#233) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2ab2020fa..19a549a02 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1566,7 +1566,7 @@ If a reactor is already installed, :meth:`CrawlerRunner.__init__ ` raises :exc:`Exception` if the installed reactor does not match the -:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported third-party libraries will make Scrapy raise :exc:`Exception` when it checks which reactor is installed. @@ -1658,7 +1658,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. This user agent is also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and -there is no overridding User-Agent header specified for the request. +there is no overriding User-Agent header specified for the request. Settings documented elsewhere: diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8c90a506c..007e9fc2f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -99,7 +99,7 @@ Available Shortcuts shortcuts - ``fetch(url[, redirect=True])`` - fetch a new response from the given URL - and update all related objects accordingly. You can optionaly ask for HTTP + and update all related objects accordingly. You can optionally ask for HTTP 3xx redirections to not be followed by passing ``redirect=False`` - ``fetch(request)`` - fetch a new response from the given request and update diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4d3d32941..99e74233a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -372,7 +372,7 @@ CrawlSpider described below. If multiple rules match the same link, the first one will be used, according to the order they're defined in this attribute. - This spider also exposes an overrideable method: + This spider also exposes an overridable method: .. method:: parse_start_url(response, **kwargs) @@ -534,7 +534,7 @@ XMLFeedSpider itertag = 'n:url' # ... - Apart from these new attributes, this spider has the following overrideable + Apart from these new attributes, this spider has the following overridable methods too: .. method:: adapt_response(response) diff --git a/docs/versioning.rst b/docs/versioning.rst index 57643ea9a..9d02757b0 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* large changes. * *B* is the release number. This will include many changes including features and things that possibly break backward compatibility, although we strive to - keep theses cases at a minimum. + keep these cases at a minimum. * *C* is the bugfix release number. Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, diff --git a/extras/qpsclient.py b/extras/qpsclient.py index f9fb70342..28703650d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -1,5 +1,5 @@ """ -A spider that generate light requests to meassure QPS troughput +A spider that generate light requests to meassure QPS throughput usage: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index f1fdc3858..c6cc7c56d 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -2,7 +2,7 @@ An extension to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -You can change the behaviour of this middleware by modifing the scraping settings: +You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fb4b565cf..36cca2d05 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -30,7 +30,7 @@ class BaseItemExporter: self._configure(kwargs, dont_fail=dont_fail) def _configure(self, options, dont_fail=False): - """Configure the exporter by poping options from the ``options`` dict. + """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index e941c4321..b5d2585a8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -88,7 +88,7 @@ class LxmlParserLinkExtractor: def _process_links(self, links): """ Normalize and filter extracted links - The subclass should override it if neccessary + The subclass should override it if necessary """ return self._deduplicate_if_needed(links) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8766ef66f..5c52c6c28 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -85,7 +85,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings + POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } @@ -142,7 +142,7 @@ class S3FilesStore: **extra) def _headers_to_botocore_kwargs(self, headers): - """ Convert headers to botocore keyword agruments. + """ Convert headers to botocore keyword arguments. """ # This is required while we need to support both boto and botocore. mapping = CaselessDict({ @@ -190,7 +190,7 @@ class GCSFilesStore: CACHE_CONTROL = 'max-age=172800' # The bucket's default object ACL will be applied to the object. - # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None def __init__(self, uri): @@ -291,7 +291,7 @@ class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading This pipeline tries to minimize network transfers and file processing, - doing stat of the files and determining if file is new, uptodate or + doing stat of the files and determining if file is new, up-to-date or expired. ``new`` files are those that pipeline never processed and needs to be diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6ed17e4dd..bef2d6b24 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -43,7 +43,7 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response, selector): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" if hasattr(self, 'parse_item'): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError @@ -113,7 +113,7 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response, row): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" raise NotImplementedError def parse_rows(self, response): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 133261fd7..1bc0bd45f 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - # Always use .instace() to ensure _instance propagation to all parents + # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports # and clear the instance to always have the fresh env # on repeated breaks like with inspect_response() diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e31284a7f..47df8a717 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -41,7 +41,7 @@ class CaselessDict(dict): return key.lower() def normvalue(self, value): - """Method to normalize values prior to be setted""" + """Method to normalize values prior to be set""" return value def get(self, key, def_val=None): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index b317c12a3..b02bfdccb 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -34,7 +34,7 @@ def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 57dcc5f2c..70ef3ba2b 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -48,7 +48,7 @@ def request_fingerprint( the fingerprint. For this reason, request headers are ignored by default when calculating - the fingeprint. If you want to include specific headers use the + the fingerprint. If you want to include specific headers use the include_headers argument, which is a list of Request headers to include. Also, servers usually ignore fragments in urls when handling requests, @@ -78,7 +78,7 @@ def request_fingerprint( def request_authenticate(request: Request, username: str, password: str) -> None: - """Autenticate the given request (in place) using the HTTP basic access + """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 00226283f..f704e113f 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -260,7 +260,7 @@ ItemForm ia['width'] = x.x('//p[@class="width"]') ia['volume'] = x.x('//p[@class="volume"]') - # another example passing parametes on instance + # another example passing parameters on instance ia = NewsForm(response, encoding='utf-8') ia['name'] = x.x('//p[@class="name"]') diff --git a/sep/sep-005.rst b/sep/sep-005.rst index e795838e4..08ed367b3 100644 --- a/sep/sep-005.rst +++ b/sep/sep-005.rst @@ -107,7 +107,7 @@ gUsing default_builder This will use default_builder as the builder for every field in the item class. -As a reducer is not set reducers will be set based on Item Field classess. +As a reducer is not set reducers will be set based on Item Field classes. gReset default_builder for a field ================================== diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..0859e3f7c 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -64,7 +64,7 @@ Request Processors takes requests objects and can perform any action to them, like filtering or modifying on the fly. The current ``LinkExtractor`` had integrated link processing, like -canonicalize. Request Processors can be reutilized and applied in serie. +canonicalize. Request Processors can be reutilized and applied in series. Request Generator ----------------- diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 372429791..c1ec16f7f 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -22,7 +22,7 @@ Instead, the hooks are spread over: * Downloader handlers (DOWNLOADER_HANDLERS) * Item pipelines (ITEM_PIPELINES) * Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) +* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) * Generic extensions (EXTENSIONS) * CLI commands (COMMANDS_MODULE) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index c376a46cd..0ec5257e1 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -19,7 +19,7 @@ class BaseResponseTest(unittest.TestCase): response_class = Response def test_init(self): - # Response requires url in the consturctor + # Response requires url in the constructor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) self.assertRaises(TypeError, self.response_class, b"http://example.com") @@ -392,7 +392,7 @@ class TextResponseTest(BaseResponseTest): def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" r = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=UKNOWN"]}, + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) self._assert_response_values(r, 'utf-8', "\xa3") diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 00c532c41..25d9657d5 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -106,9 +106,9 @@ class CrawlTestCase(TestCase): """ Downloader middleware which returns a response with an specific 'request' attribute. - * The spider callback should receive the overriden response.request - * Handlers listening to the response_received signal should receive the overriden response.request - * The "crawled" log message should show the overriden response.request + * The spider callback should receive the overridden response.request + * Handlers listening to the response_received signal should receive the overridden response.request + * The "crawled" log message should show the overridden response.request """ signal_params = {} @@ -144,7 +144,7 @@ class CrawlTestCase(TestCase): An exception is raised but caught by the next middleware, which returns a Response with a specific 'request' attribute. - The spider callback should receive the overriden response.request + The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 7a5f458c7..032dbc8c5 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -23,7 +23,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd def test_unfired_deferred(self): @@ -37,7 +37,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5ff2e41ef..1d5e63363 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -36,7 +36,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): self.assertEqual(result.read().decode('utf8'), rendered) os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test iself + assert not os.path.exists(render_path) # Failure of test itself if '__main__' == __name__: From d08199f631814bdaadafc441bb47cbe71ced2d8d Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Tue, 12 Oct 2021 13:20:09 -0400 Subject: [PATCH 1816/1853] Removing unnecessary line from docs to prevent test failure (#5274) --- docs/intro/tutorial.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fa321a770..ca5856881 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -282,7 +282,6 @@ raise an :exc:`IndexError` exception if there are no results:: >>> response.css('noelement')[0].get() Traceback (most recent call last): - File "", line 1, in ... IndexError: list index out of range From 3243aa2cd54c8789eb5667998aa8296f6adaa9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E4=B9=9D=E9=BC=8E?= <109224573@qq.com> Date: Thu, 14 Oct 2021 10:18:26 +0800 Subject: [PATCH 1817/1853] docs: fix typo --- docs/topics/loaders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index c0f534493..0d63700c8 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -56,7 +56,7 @@ chapter `:: l.add_xpath('name', '//div[@class="product_name"]') l.add_xpath('name', '//div[@class="product_title"]') l.add_xpath('price', '//p[@id="price"]') - l.add_css('stock', 'p#stock]') + l.add_css('stock', 'p#stock') l.add_value('last_updated', 'today') # you can also use literal values return l.load_item() From ca320feb2afa5eae3990ed21df6a8df930e8cf9f Mon Sep 17 00:00:00 2001 From: Erik Kemperman Date: Fri, 15 Oct 2021 15:43:55 +0200 Subject: [PATCH 1818/1853] Add LOG_FILE_APPEND to settings --- docs/topics/logging.rst | 5 ++++- docs/topics/settings.rst | 10 ++++++++++ scrapy/settings/default_settings.py | 1 + scrapy/utils/log.py | 3 ++- tests/test_crawler.py | 29 ++++++++++++++++++++++++++++- 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index dda04dc4d..d593c74c6 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -143,6 +143,7 @@ Logging settings These settings can be used to configure the logging: * :setting:`LOG_FILE` +* :setting:`LOG_FILE_APPEND` * :setting:`LOG_ENABLED` * :setting:`LOG_ENCODING` * :setting:`LOG_LEVEL` @@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be redirected to a file named :setting:`LOG_FILE` with encoding :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log -messages will be displayed on the standard error. Lastly, if +messages will be displayed on the standard error. If :setting:`LOG_FILE` is set +and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten +(discarding the output from previous runs, if any). Lastly, if :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output. :setting:`LOG_LEVEL` determines the minimum level of severity to display, those diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e63aca312..210c1def7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1026,6 +1026,16 @@ Default: ``None`` File name to use for logging output. If ``None``, standard error will be used. +.. setting:: LOG_FILE_APPEND + +LOG_FILE_APPEND +--------------- + +Default: ``True`` + +If ``False``, the log file specified with :setting:`LOG_FILE` will be +overwritten (discarding the output from previous runs, if any). + .. setting:: LOG_FORMAT LOG_FORMAT diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4ef330dd2..8389a70cb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_FILE_APPEND = True LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6c456ed60..0441c0358 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -124,8 +124,9 @@ def _get_handler(settings): """ Return a log handler object according to settings """ filename = settings.get('LOG_FILE') if filename: + mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w' encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) + handler = logging.FileHandler(filename, mode=mode, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index dec517bb6..a80ad4388 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -98,6 +98,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) class MySpider(scrapy.Spider): name = 'spider' @@ -119,8 +121,9 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.error('error message') with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf8') + logged = fo.read().decode('utf-8') + self.assertIn('previous message', logged) self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -131,6 +134,30 @@ class CrawlerLoggingTestCase(unittest.TestCase): crawler.stats.get_value('log_count/INFO') - info_count, 1) self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + def test_spider_custom_settings_log_append(self): + log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) + + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_FILE': log_file, + 'LOG_FILE_APPEND': False, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + crawler = Crawler(MySpider, {}) + logging.debug('debug message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf-8') + + self.assertNotIn('previous message', logged) + self.assertIn('debug message', logged) + class SpiderLoaderWithWrongInterface: From 98ee3ddffeeabdc896c3f2cffce8310ebfc97570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Oct 2021 17:18:46 +0200 Subject: [PATCH 1819/1853] Freeze flake8 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e274fc8d2..07552ba8d 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,7 @@ deps = # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 pytest-flake8 + flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = py.test --flake8 {posargs:docs scrapy tests} From d774d6a9c46cb9697d8fe64a55da9ae321be7539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Oct 2021 17:25:22 +0200 Subject: [PATCH 1820/1853] Remove unused variable --- tests/test_crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a80ad4388..be067155e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -149,7 +149,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): } configure_logging() - crawler = Crawler(MySpider, {}) + Crawler(MySpider, {}) logging.debug('debug message') with open(log_file, 'rb') as fo: From aec7146e2f870f5e8f0f58bd596b1ec40c64b3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Raphael=20Tom=C3=A9=20Santana?= Date: Fri, 15 Oct 2021 20:38:53 -0300 Subject: [PATCH 1821/1853] Add how Scrapy is pronounced to the docs --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index dd80c7bd0..d75f7f636 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy is an application framework for crawling web sites and extracting +Scrapy (pronounced SKRAY-peye /ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. From 027ecd8686d7da74b73412e4aa38d8be36b5b9f1 Mon Sep 17 00:00:00 2001 From: raphaelts3 Date: Sat, 16 Oct 2021 10:52:54 -0300 Subject: [PATCH 1822/1853] Update docs/intro/overview.rst Co-authored-by: azzamsa <17734314+azzamsa@users.noreply.github.com> --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d75f7f636..405bf845d 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy (pronounced SKRAY-peye /ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting +Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. From cfff79cee6a97528185b7d24e2b660b99c07945f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 18 Oct 2021 17:09:17 -0300 Subject: [PATCH 1823/1853] Make Python 3.10 support official (#5265) --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 14 ++++++-------- .readthedocs.yml | 9 +++++---- setup.py | 1 + tox.ini | 1 + 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6bdfcb5dc..80df9469d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,10 +8,10 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + - python-version: "3.10" env: TOXENV: security - - python-version: 3.9 + - python-version: "3.10" env: TOXENV: flake8 # Pylint requires installing reppy, which does not support Python 3.9 @@ -20,10 +20,10 @@ jobs: env: TOXENV: pylint TOX_PIP_VERSION: 20.3.3 - - python-version: 3.9 + - python-version: 3.6 env: TOXENV: typing - - python-version: 3.8 # Keep in sync with .readthedocs.yml + - python-version: "3.10" # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b48066ea4..44b682830 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 095ca1013..3aaf688c7 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ef1c8362f..5ea50e644 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,6 +17,12 @@ jobs: - python-version: 3.9 env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio - python-version: pypy3 env: TOXENV: pypy3 @@ -42,14 +48,6 @@ jobs: TOXENV: extra-deps TOX_PIP_VERSION: 20.3.3 - # 3.10-pre - - python-version: "3.10.0-beta.4" - env: - TOXENV: py - - python-version: "3.10.0-beta.4" - env: - TOXENV: asyncio - steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index 80a1cd036..390be3749 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,12 +5,13 @@ sphinx: fail_on_warning: true build: - image: latest + os: ubuntu-20.04 + tools: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python + python: "3.10" # Keep in sync with .github/workflows/checks.yml python: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/setup.py b/setup.py index ed2b6e347..3a6ff2836 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ setup( 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tox.ini b/tox.ini index 07552ba8d..021dd9988 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,7 @@ deps = types-pyOpenSSL==20.0.3 types-setuptools==57.0.0 commands = + pip install types-dataclasses # remove once py36 support is dropped mypy --show-error-codes {posargs: scrapy tests} [testenv:security] From 144d1eb8341c427fa1fae109db3e9487f255d3fe Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 22 Oct 2021 21:46:01 +0500 Subject: [PATCH 1824/1853] Add Deferred-to-Future helpers (#5288) --- conftest.py | 6 ++++ docs/topics/asyncio.rst | 17 ++++++++++ docs/topics/coroutines.rst | 13 +++++--- docs/topics/item-pipeline.rst | 4 ++- pytest.ini | 1 + scrapy/utils/defer.py | 63 +++++++++++++++++++++++++++++++++-- tests/test_pipelines.py | 30 ++++++++++++++++- 7 files changed, 126 insertions(+), 8 deletions(-) diff --git a/conftest.py b/conftest.py index 05b4ccdad..117087790 100644 --- a/conftest.py +++ b/conftest.py @@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 82c5f271f..28241ae24 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -39,5 +39,22 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. +.. _asyncio-await-dfd: +Awaiting on Deferreds +===================== +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +:class:`asyncio.Future` objects, not into +:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future +.. tip:: If you need to use these functions in code that aims to be compatible + with lower versions of Scrapy that do not provide these functions, + down to Scrapy 2.0 (earlier versions do not support + :mod:`asyncio`), you can copy the implementation of these functions + into your own code. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 0904637b0..2aef755c7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -75,23 +75,28 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code:: - class MySpider(Spider): + class MySpiderDeferred(Spider): # ... - async def parse_with_deferred(self, response): + async def parse(self, response): additional_response = await treq.get('https://additional.url') additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests - async def parse_with_asyncio(self, response): + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: - additional_data = await r.text() + additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the :mod:`asyncio` loop and to use them you need to :doc:`enable asyncio support in Scrapy`. +.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor, + you need to :ref:`wrap them`. + Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in callbacks, diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 5351a2293..391751364 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -190,6 +190,8 @@ item. import scrapy from itemadapter import ItemAdapter + from scrapy.utils.defer import maybe_deferred_to_future + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of @@ -202,7 +204,7 @@ item. encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) - response = await spider.crawler.engine.download(request, spider) + response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider)) if response.status != 200: # Error happened, return item. diff --git a/pytest.ini b/pytest.ini index 6de08c78d..fa5d6b34f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,5 +20,6 @@ addopts = --ignore=docs/utils markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed + only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed filterwarnings= ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index b02bfdccb..d7adc0a77 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect -from collections.abc import Coroutine +from asyncio import Future from functools import wraps -from typing import Any, Callable, Generator, Iterable +from typing import ( + Any, + Callable, + Coroutine, + Generator, + Iterable, + Union +) from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: return defer.fail(result) else: return defer.succeed(result) + + +def deferred_to_future(d: Deferred) -> Future: + """ + .. versionadded:: VERSION + + Return an :class:`asyncio.Future` object that wraps *d*. + + When :ref:`using the asyncio reactor `, you cannot await + on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy + callables defined as coroutines `, you can only await on + ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects + allows you to wait on them:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + additional_response = await deferred_to_future(d) + """ + return d.asFuture(asyncio.get_event_loop()) + + +def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: + """ + .. versionadded:: VERSION + + Return *d* as an object that can be awaited from a :ref:`Scrapy callable + defined as a coroutine `. + + What you can await in Scrapy callables defined as coroutines depends on the + value of :setting:`TWISTED_REACTOR`: + + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + + - When :ref:`using the asyncio reactor `, you can only + await on :class:`asyncio.Future` objects. + + If you want to write code that uses ``Deferred`` objects but works with any + reactor, use this function on all ``Deferred`` objects:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + extra_response = await maybe_deferred_to_future(d) + """ + if not is_asyncio_reactor_installed(): + return d + else: + return deferred_to_future(d) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index ff3af9a74..8e432b913 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -6,6 +6,7 @@ from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy import Spider, signals, Request +from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -31,18 +32,38 @@ class DeferredPipeline: class AsyncDefPipeline: async def process_item(self, item, spider): - await defer.succeed(42) + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await maybe_deferred_to_future(d) item['pipeline_passed'] = True return item class AsyncDefAsyncioPipeline: async def process_item(self, item, spider): + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await deferred_to_future(d) await asyncio.sleep(0.2) item['pipeline_passed'] = await get_from_asyncio_queue(True) return item +class AsyncDefNotAsyncioPipeline: + async def process_item(self, item, spider): + d1 = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d1.callback, None) + await d1 + d2 = Deferred() + reactor.callLater(0, d2.callback, None) + await maybe_deferred_to_future(d2) + item['pipeline_passed'] = True + return item + + class ItemSpider(Spider): name = 'itemspider' @@ -99,3 +120,10 @@ class PipelineTestCase(unittest.TestCase): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) + + @mark.only_not_asyncio() + @defer.inlineCallbacks + def test_asyncdef_not_asyncio_pipeline(self): + crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) From 51adf71b1b4ae703bb1cb561882c710eedac2359 Mon Sep 17 00:00:00 2001 From: azzamsa Date: Sun, 24 Oct 2021 10:52:56 +0700 Subject: [PATCH 1825/1853] refactor: use `pytest` command as the recommended entry point `pytest` is recommended command since pytest 3.0. There is a possibility for `py.test` to be deprecated or even removed. https://github.com/pytest-dev/pytest/issues/1629 --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 021dd9988..e4514f512 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = pip install -U -ctests/upper-constraints.txt {opts} {packages} @@ -60,7 +60,7 @@ deps = pytest-flake8 flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = - py.test --flake8 {posargs:docs scrapy tests} + pytest --flake8 {posargs:docs scrapy tests} [testenv:pylint] basepython = python3 @@ -142,7 +142,7 @@ setenv = [testenv:pypy3] basepython = pypy3 commands = - py.test {posargs:--durations=10 docs scrapy tests} + pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} From 67994d1dddcba4c1fe53dd7bdf7b978d1733ae1b Mon Sep 17 00:00:00 2001 From: azzamsa Date: Wed, 27 Oct 2021 21:55:05 +0700 Subject: [PATCH 1826/1853] fix: `CodeBlockParser` has been renamed to `PythonCodeBlockParser` --- docs/conftest.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/conftest.py b/docs/conftest.py index 8c735e838..a0636f8ac 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from scrapy.http.response.html import HtmlResponse from sybil import Sybil -from sybil.parsers.codeblock import CodeBlockParser +try: + # >2.0.1 + from sybil.parsers.codeblock import PythonCodeBlockParser +except ImportError: + from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip @@ -21,7 +25,7 @@ def setup(namespace): pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - CodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=['print_function']), skip, ], pattern='*.rst', From 55cce25a799ab0ed9d5edd60ff0f35988318e9b9 Mon Sep 17 00:00:00 2001 From: azzamsa Date: Mon, 25 Oct 2021 21:14:11 +0700 Subject: [PATCH 1827/1853] test: `test_format_engine_status` --- tests/test_crawl.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 84bac9b50..7bda3bef2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -274,6 +274,28 @@ with multiples lines self.assertEqual(s['engine.spider.name'], crawler.spider.name) self.assertEqual(s['len(engine.scraper.slot.active)'], 1) + @defer.inlineCallbacks + def test_format_engine_status(self): + from scrapy.utils.engine import format_engine_status + est = [] + + def cb(response): + est.append(format_engine_status(crawler.engine)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) + self.assertEqual(len(est), 1, est) + est = est[0].split("\n")[2:-2] # remove header & footer + # convert to dict + est = [x.split(":") for x in est] + est = [x for sublist in est for x in sublist] # flatten + est = [x.lstrip().rstrip() for x in est] + it = iter(est) + s = dict(zip(it, it)) + + self.assertEqual(s['engine.spider.name'], crawler.spider.name) + self.assertEqual(s['len(engine.scraper.slot.active)'], '1') + @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): """ From 28eba610e22c0d2a42e830b4e64746edf44598f9 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 15 Nov 2021 12:24:54 +0500 Subject: [PATCH 1828/1853] Re-enable Windows tests for Python 3.9 and 3.10. (#5316) --- .github/workflows/tests-windows.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 30fda33e8..6fabf5cde 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,10 +17,12 @@ jobs: - python-version: 3.8 env: TOXENV: py - # https://twistedmatrix.com/trac/ticket/9990 - #- python-version: 3.9 - #env: - #TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py steps: - uses: actions/checkout@v2 From f2c800c5c9f88b4b583181e9cf49eb3cd8d538f0 Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 11:14:54 +0100 Subject: [PATCH 1829/1853] Improve open_in_browser base tag injection (#5319) --- scrapy/utils/response.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3ef7b463..8b109dced 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,8 +3,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import webbrowser +import re import tempfile +import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -80,8 +81,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' From 75ed765476a2ac66ea8f52e7b29186864f65535c Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 14:31:24 +0100 Subject: [PATCH 1830/1853] Test coverage for open_in_browser base tag injection (#5319) --- tests/test_utils_response.py | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

                Hello world.

                + """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
                Hello header
                +

                Hello world.

                + + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

                Hello world.

                + """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

                Hello world.

                + """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" From c316ca45a5b1b19622c96049c9378d8c45adba60 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 16 Nov 2021 01:20:56 -0800 Subject: [PATCH 1831/1853] Use augmented assignment statements (#5322) --- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/core/http2/stream.py | 4 ++-- tests/test_request_left.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8a91d4c5e..38935667d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -213,7 +213,7 @@ class TunnelingAgent(Agent): # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy - key = key + self._proxyConf + key += self._proxyConf return super()._requestWithEndpoint( key=key, endpoint=endpoint, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2a4b702f..5c393c027 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -285,8 +285,8 @@ class Stream: self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) - bytes_to_send_size = bytes_to_send_size - chunk_size - self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size + bytes_to_send_size -= chunk_size + self.metadata['remaining_content_length'] -= chunk_size self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 373b2e49c..4d4483881 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider): return spider def on_request_left(self, request, spider): - self.caught_times = self.caught_times + 1 + self.caught_times += 1 class TestCatching(TestCase): From 6ec66c96fb962a276c2d285cd5ffa34d84925df1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 12:25:45 +0500 Subject: [PATCH 1832/1853] Fix and pin pylint. --- pylintrc | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 699686e16..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -112,6 +112,7 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/tox.ini b/tox.ini index e4514f512..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint + pylint==2.12.1 commands = pylint conftest.py docs extras scrapy setup.py tests From 4cc039628eb4861f98f7997e90125745e30f8687 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 19:52:03 +0500 Subject: [PATCH 1833/1853] Fix typing of middleware methods. --- scrapy/core/downloader/middleware.py | 5 ++++- scrapy/core/spidermw.py | 3 ++- scrapy/middleware.py | 16 ++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a5619d8a4..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union +from typing import Callable, Union, cast from twisted.internet import defer from twisted.python.failure import Failure @@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7e58521ac..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union +from typing import Any, Callable, Generator, Iterable, Union, cast from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Any: for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: diff --git a/scrapy/middleware.py b/scrapy/middleware.py index bbec38086..e8f60287a 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict +from typing import Callable, Deque, Dict, Optional, cast, Iterable from twisted.internet.defer import Deferred @@ -21,7 +21,8 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -64,14 +65,17 @@ class MiddlewareManager: self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: - return process_parallel(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) def _process_chain(self, methodname: str, obj, *args) -> Deferred: - return process_chain(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) + cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) + eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) + return process_chain_both(cb_methods, eb_methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From eb62906c3e4c1e1f8e3e6c7965a04d5e65c61907 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Dec 2021 17:40:41 +0500 Subject: [PATCH 1834/1853] Extract utils.log.log_reactor_info(). --- scrapy/utils/log.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0441c0358..9887ecc40 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -152,6 +152,10 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + log_reactor_info() + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor From 6483dfdbe17cd66c409435b95a05850a3c94b5ee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Dec 2021 19:53:39 +0500 Subject: [PATCH 1835/1853] Move install_shutdown_handlers() from __init__() to start(). --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..357f14dc0 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -278,7 +278,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -318,6 +317,7 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() From d6a384b3cfdb36cd19b32942479487a9e47c244b Mon Sep 17 00:00:00 2001 From: yogender26 <95638485+yogender26@users.noreply.github.com> Date: Thu, 23 Dec 2021 04:09:05 +0530 Subject: [PATCH 1836/1853] corrrection of coma (#5347) --- scrapy/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 6e77551c6..5f1dabd33 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -43,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() From 46ef9cf771789f1db513bbf2f65243d3320ce695 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 22 Dec 2021 21:24:59 +0500 Subject: [PATCH 1837/1853] Don't install non-working shutdown handlers in `scrapy shell`. --- scrapy/commands/shell.py | 2 +- scrapy/crawler.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..de81986d8 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 357f14dc0..e54ad9750 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -297,7 +297,7 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -308,6 +308,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -317,7 +320,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - install_shutdown_handlers(self._signal_shutdown) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() From 60c8838554a79e70c22a7c6a57baedfcaf521444 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:07:18 +0500 Subject: [PATCH 1838/1853] Move installing the reactor from CrawlerProcess to Crawler. --- scrapy/crawler.py | 31 ++++++++++++++++++++----------- scrapy/utils/log.py | 1 - tests/test_crawler.py | 5 ++++- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e54ad9750..95cfb1bd1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -69,6 +70,19 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if self.settings.get("TWISTED_REACTOR"): + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if self.settings.get("TWISTED_REACTOR"): + verify_installed_reactor(self.settings["TWISTED_REACTOR"]) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +167,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +260,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -297,6 +306,11 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool @@ -341,8 +355,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 9887ecc40..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -152,7 +152,6 @@ def log_scrapy_info(settings: Settings) -> None: if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) - log_reactor_info() def log_reactor_info() -> None: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..118cb631b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -271,6 +271,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -279,9 +280,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) + yield runner.crawl(NoRequestsSpider) @defer.inlineCallbacks # https://twistedmatrix.com/trac/ticket/9766 @@ -301,6 +303,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): runner = CrawlerProcess(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) + yield runner.crawl(NoRequestsSpider) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_false(self): From 041699b54cfa6cde9f886a98ff300e3276e2eaad Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:14:47 +0500 Subject: [PATCH 1839/1853] Remove tests that want to modify the test process reactor. --- tests/test_crawler.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 118cb631b..f445c181e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -285,33 +285,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) yield runner.crawl(NoRequestsSpider) - @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - yield runner.crawl(NoRequestsSpider) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - class ScriptRunnerMixin: def run_script(self, script_name, *script_args): From ebcafdf4a9e0692bf301546b6d60465b3b2c4b06 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:35:26 +0500 Subject: [PATCH 1840/1853] Add tests for TWISTED_REACTOR in custom_settings. --- .../twisted_reactor_custom_settings.py | 14 +++++++++++ ...wisted_reactor_custom_settings_conflict.py | 22 +++++++++++++++++ .../twisted_reactor_custom_settings_same.py | 21 ++++++++++++++++ tests/test_crawler.py | 24 +++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings.py create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_same.py diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..9a6c01d72 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class PollReactorSpider(scrapy.Spider): + name = 'poll_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(PollReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..1f5a44010 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f445c181e..6d6763aec 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') From 002513438204eea5062b5a1d75fb4f261880da4f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:45:17 +0500 Subject: [PATCH 1841/1853] Completely skip WindowsRunSpiderCommandTest outside Windows. --- tests/test_commands.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..efe9b0531 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -765,6 +765,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -777,35 +778,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() From 9c4bfb48362f736fce81b71a6ca1fa0b3600231d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 17:17:36 +0500 Subject: [PATCH 1842/1853] Remove an unused import. --- tests/test_crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6d6763aec..d68c50026 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -7,7 +7,6 @@ import warnings from unittest import skipIf from pytest import raises, mark -from testfixtures import LogCapture from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version From d4565318c7061c2ccd17fa5d5eabcacef8c34826 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 17:40:31 +0500 Subject: [PATCH 1843/1853] Fix a reactor test on Windows. --- .../twisted_reactor_custom_settings_conflict.py | 8 ++++---- tests/test_crawler.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 9a6c01d72..3f219098c 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -2,10 +2,10 @@ import scrapy from scrapy.crawler import CrawlerProcess -class PollReactorSpider(scrapy.Spider): - name = 'poll_reactor' +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' custom_settings = { - "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", } @@ -17,6 +17,6 @@ class AsyncioReactorSpider(scrapy.Spider): process = CrawlerProcess() -process.crawl(PollReactorSpider) +process.crawl(SelectReactorSpider) process.crawl(AsyncioReactorSpider) process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d68c50026..e7d5c8132 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -381,8 +381,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') From 940cc0776ff86f726e79c2ab2018f4b83a833936 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Dec 2021 17:12:50 +0500 Subject: [PATCH 1844/1853] Add docs about TWISTED_REACTOR and other per-process settings. --- docs/topics/practices.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..bd0dd8ce0 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -102,6 +102,17 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + .. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: @@ -193,6 +204,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: From a986792def6df1b2bbdf1bc996308d3afd8528c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Dec 2021 19:43:14 +0500 Subject: [PATCH 1845/1853] Add more docs for TWISTED_REACTOR. --- docs/topics/settings.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 210c1def7..cff6d80cb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1638,10 +1638,18 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform will be used. This is to maintain backward compatibility and +avoid possible problems caused by using a non-default reactor. + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. For additional information, see :doc:`core/howto/choosing-reactor`. From 7380888cad2c255e6f4a7efb6fe4cfd1240fb42c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 30 Dec 2021 18:55:16 +0500 Subject: [PATCH 1846/1853] Fix a warning message. (#5359) --- scrapy/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 24873f75d..24a6187b9 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From 64261d9e389737621caa85f320cf81ef2aef1faa Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 15:45:59 +0500 Subject: [PATCH 1847/1853] Slight refactoring. --- scrapy/crawler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 95cfb1bd1..a638254f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,17 +71,18 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + reactor_class = self.settings.get("TWISTED_REACTOR") if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) else: from twisted.internet import default default.install() log_reactor_info() - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) + if reactor_class: + verify_installed_reactor(reactor_class) self.extensions = ExtensionManager.from_crawler(self) From b81938684b55fca29e48faa766f2b6f6e3ab5d6a Mon Sep 17 00:00:00 2001 From: Andrey Oskin Date: Fri, 31 Dec 2021 21:49:18 +1100 Subject: [PATCH 1848/1853] Docs: correct process repetition start step (#5356) The process repeats from step 3, the scheduler feeds request to the engine. Steps 1 and 2 are not parts of the loop as their incarnations steps 7 and 8 are parts of the loop. --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 71d027c86..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components From e4bdd1cb958b7d89b86ea66f0af1cec2d91a6d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Miech?= Date: Fri, 31 Dec 2021 11:57:12 +0100 Subject: [PATCH 1849/1853] downloader.webclient: make reactor import local (#5357) --- scrapy/core/downloader/webclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 915cb5fe3..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers @@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p From 57dc58123b98e2026025cc87bdee474bf0656dcb Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 31 Dec 2021 17:15:08 +0500 Subject: [PATCH 1850/1853] Remove the experimental note about asyncio (#5332) --- docs/topics/asyncio.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 28241ae24..402352721 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet - recommended for production environments. Future Scrapy versions - may introduce related changes without a deprecation period or - warning. - .. _install-asyncio: Installing the asyncio reactor From a2763c608d33a9254034d650457b7efa7b434ec0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 18:35:00 +0500 Subject: [PATCH 1851/1853] Remove unused MiddlewareManager._process_chain_both(). --- scrapy/middleware.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index e8f60287a..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -72,11 +72,6 @@ class MiddlewareManager: methods = cast(Iterable[Callable], self.methods[methodname]) return process_chain(methods, obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) - eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) - return process_chain_both(cb_methods, eb_methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From 6eaceec735d551f5b777bc641ff8d85dbb3ba98c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 20:14:24 +0500 Subject: [PATCH 1852/1853] Implement docs suggestions. --- docs/news.rst | 22 ++++++++++++++++++++++ docs/topics/practices.rst | 11 ----------- docs/topics/settings.rst | 13 ++----------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 509366c17..2afe318f6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,3 +1,25 @@ +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. + + .. _news: Release notes diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index bd0dd8ce0..1a9d56143 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -102,17 +102,6 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -.. note:: - .. versionchanged:: VERSION - - The Twisted reactor is now installed when - :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a - :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, - :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now - honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions - they are silently ignored when set there and you need to set these settings - in some other way. - .. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cff6d80cb..f6c95c502 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1639,17 +1639,8 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which means that Scrapy will install the default reactor defined by Twisted for the -current platform will be used. This is to maintain backward compatibility and -avoid possible problems caused by using a non-default reactor. - -.. note:: - .. versionchanged:: VERSION - - Previously this setting had no effect in a spider - :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but - if you :ref:`run several spiders in one process `, - they must not have different values for this setting, because they will use - a single reactor instance. +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. From c5ab58056c29c2c35b183572b0780acfbb15dfe8 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Sat, 1 Jan 2022 00:38:10 +0500 Subject: [PATCH 1853/1853] Set WindowsSelectorEventLoopPolicy on Windows (#5315) --- .github/workflows/tests-windows.yml | 3 ++ docs/topics/asyncio.rst | 28 +++++++++++++++++++ scrapy/utils/reactor.py | 5 ++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 3 ++ tests/test_commands.py | 11 +++----- tests/test_crawler.py | 16 ----------- tests/test_downloader_handlers.py | 16 +++++++++++ tests/test_utils_asyncio.py | 7 +---- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 6fabf5cde..ab7385118 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,9 @@ jobs: - python-version: "3.10" env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 402352721..8712d4268 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -36,6 +36,34 @@ use it instead of the default asyncio event loop. .. _asyncio-await-dfd: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: :class:`~asyncio.SelectorEventLoop` (default before Python +3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` +(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the +event loop class needs to be changed. Scrapy since VERSION does this +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor +by other means or use an older Scrapy version you need to call the following +code before installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + Awaiting on Deferreds ===================== diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..81d1a1cab 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -674,9 +674,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -699,15 +696,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..7bc4fba40 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,7 +4,6 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -284,9 +283,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -328,17 +324,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -377,9 +367,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) @@ -404,9 +391,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9c11820e5..a1ea4c679 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import contextlib import os import shutil +import sys import tempfile from typing import Optional, Type from unittest import mock @@ -287,6 +288,12 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -296,6 +303,11 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -1055,6 +1067,10 @@ class BaseFTPTestCase(unittest.TestCase): class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")