diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 97f8311de..a93bee06b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -461,6 +461,9 @@ Supported options: * ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' +* ``--cbkwargs``: additional keyword arguments that will be passed to the callback. + This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index f93aa2c72..0aaad0c77 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -28,16 +28,15 @@ Consider the following scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item}) + yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) - def parse_details(self, response): - item = response.meta['item'] + def parse_details(self, response, item): # populate more `item` fields return item Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a partially populated item. @@ -100,8 +99,7 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see from scrapy.shell import inspect_response - def parse_details(self, response): - item = response.meta.get('item', None) + def parse_details(self, response, item=None): if item: # populate more `item` fields return item @@ -134,8 +132,7 @@ Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be available in all future runs should they be necessary again:: - def parse_details(self, response): - item = response.meta.get('item', None) + def parse_details(self, response, item=None): if item: # populate more `item` fields return item diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 1a5d52487..9fd311c69 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -81,7 +81,8 @@ So, for example, this won't work:: def some_callback(self, response): somearg = 'test' - return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg)) + return scrapy.Request('http://www.example.com', + callback=lambda r: self.other_callback(r, somearg)) def other_callback(self, response, somearg): print("the argument passed is: %s" % somearg) @@ -90,10 +91,10 @@ But this will:: def some_callback(self, response): somearg = 'test' - return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg}) + return scrapy.Request('http://www.example.com', + callback=self.other_callback, cb_kwargs={'somearg': somearg}) - def other_callback(self, response): - somearg = response.meta['somearg'] + def other_callback(self, response, somearg): print("the argument passed is: %s" % somearg) If you wish to log the requests that couldn't be serialized, you can set the diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index af14d14e8..8278e9849 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -27,10 +27,11 @@ Common causes of memory leaks It happens quite often (sometimes by accident, sometimes on purpose) that the Scrapy developer passes objects referenced in Requests (for example, using the -:attr:`~scrapy.http.Request.meta` attribute or the request callback function) -and that effectively bounds the lifetime of those referenced objects to the -lifetime of the Request. This is, by far, the most common cause of memory leaks -in Scrapy projects, and a quite difficult one to debug for newcomers. +:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta` +attributes or the request callback function) and that effectively bounds the +lifetime of those referenced objects to the lifetime of the Request. This is, +by far, the most common cause of memory leaks in Scrapy projects, and a quite +difficult one to debug for newcomers. In big projects, the spiders are typically written by different people and some of those spiders could be "leaking" and thus affecting the rest of the other @@ -48,7 +49,8 @@ Too Many Requests? By default Scrapy keeps the request queue in memory; it includes :class:`~scrapy.http.Request` objects and all objects -referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`). +referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs` +and :attr:`~scrapy.http.Request.meta`). While not necessarily a leak, this can take a lot of memory. Enabling :ref:`persistent job queue ` could help keeping memory usage in control. @@ -101,7 +103,7 @@ Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, - callback=self.parse, meta={referer: response}) + callback=self.parse, cb_kwargs={'referer': response}) That line is passing a response reference inside a request which effectively ties the response lifetime to the requests' one, and that would definitely diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ac6fe6e3f..4e81ce878 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags]) +.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -126,6 +126,9 @@ Request objects :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list + :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback. + :type cb_kwargs: dict + .. attribute:: Request.url A string containing the URL of this request. Keep in mind that this @@ -165,6 +168,17 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. + .. attribute:: Request.cb_kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a :class:`Response` + object as argument. + + This dict is `shallow copied`_ when the request is cloned using the + ``copy()`` or ``replace()`` methods, and can also be accessed, in your + spider, from the ``response.cb_kwargs`` attribute. + .. _shallow copied: https://docs.python.org/2/library/copy.html .. method:: Request.copy() @@ -172,12 +186,12 @@ Request objects Return a new Request which is a copy of this Request. See also: :ref:`topics-request-response-ref-request-callback-arguments`. - .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback]) + .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs]) Return a Request object with the same members, except for those members given new values by whichever keyword arguments are specified. The - attribute :attr:`Request.meta` is copied by default (unless a new value - is given in the ``meta`` argument). See also + :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow + copied by default (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. .. _topics-request-response-ref-request-callback-arguments: @@ -200,25 +214,31 @@ Example:: self.logger.info("Visited %s", response.url) In some cases you may be interested in passing arguments to those callback -functions so you can receive the arguments later, in the second callback. You -can use the :attr:`Request.meta` attribute for that. +functions so you can receive the arguments later, in the second callback. +The following example shows how to achieve this by using the +:attr:`Request.cb_kwargs` attribute: -Here's an example of how to pass an item using this mechanism, to populate -different fields from different pages:: +:: - def parse_page1(self, response): - item = MyItem() - item['main_url'] = response.url - request = scrapy.Request("http://www.example.com/some_page.html", - callback=self.parse_page2) - request.meta['item'] = item + def parse(self, response): + request = scrapy.Request('http://www.example.com/index.html', + callback=self.parse_page2, + cb_kwargs=dict(main_url=response.url)) + request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback yield request - def parse_page2(self, response): - item = response.meta['item'] - item['other_url'] = response.url - yield item + def parse_page2(self, response, main_url, foo): + yield dict( + main_url=main_url, + other_url=response.url, + foo=foo, + ) +.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. + Prior to that, using :attr:`Request.meta` was recommended for passing + information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs` + became the preferred way for handling user information, leaving :attr:`Request.meta` + for communication with components like middlewares and extensions. .. _topics-request-response-ref-errbacks: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 69418a478..d4f2234b0 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,12 +51,13 @@ class Command(ScrapyCommand): help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_option("--cbkwargs", dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") - @property def max_level(self): levels = list(self.items.keys()) + list(self.requests.keys()) @@ -111,10 +112,11 @@ class Command(ScrapyCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, cb): + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} items, requests = [], [] - for x in iterate_spider_output(cb(response)): + for x in iterate_spider_output(callback(response, **cb_kwargs)): if isinstance(x, (BaseItem, dict)): items.append(x) elif isinstance(x, Request): @@ -142,8 +144,7 @@ class Command(ScrapyCommand): else: self.spidercls = spidercls_for_request(spider_loader, Request(url)) if not self.spidercls: - logger.error('Unable to find spider for: %(url)s', - {'url': url}) + logger.error('Unable to find spider for: %(url)s', {'url': url}) # Request requires callback argument as callable or None, not string request = Request(url, None) @@ -160,7 +161,7 @@ class Command(ScrapyCommand): {'url': url}) def prepare_request(self, spider, request, opts): - def callback(response): + def callback(response, **cb_kwargs): # memorize first request if not self.first_response: self.first_response = response @@ -175,7 +176,7 @@ class Command(ScrapyCommand): if not cb: logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', - {'url': response.url, 'spider': spider.name}) + {'url': response.url, 'spider': spider.name}) return else: cb = 'parse' @@ -192,7 +193,7 @@ class Command(ScrapyCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb) + items, requests = self.run_callback(response, cb, cb_kwargs) if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc for item in items: @@ -207,10 +208,14 @@ class Command(ScrapyCommand): req.callback = callback return requests - #update request meta if any extra meta was passed through the --meta/-m opts. + # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: request.meta.update(opts.meta) + # update cb_kwargs if any extra values were was passed through the --cbkwargs option. + if opts.cbkwargs: + request.cb_kwargs.update(opts.cbkwargs) + request.meta['_depth'] = 1 request.meta['_callback'] = request.callback request.callback = callback @@ -221,23 +226,29 @@ class Command(ScrapyCommand): self.process_spider_arguments(opts) self.process_request_meta(opts) + self.process_request_cb_kwargs(opts) def process_spider_arguments(self, opts): - try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) def process_request_meta(self, opts): - if opts.meta: try: opts.meta = json.loads(opts.meta) except ValueError: - raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \ - "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " + "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + def process_request_cb_kwargs(self, opts): + if opts.cbkwargs: + try: + opts.cbkwargs = json.loads(opts.cbkwargs) + except ValueError: + raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " + "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False) def run(self, args, opts): # parse arguments diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d7fe721fb..66f5d0e05 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -142,7 +142,9 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - dfd.addCallbacks(request.callback or spider.parse, request.errback) + dfd.addCallbacks(callback=request.callback or spider.parse, + errback=request.errback, + callbackKeywords=request.cb_kwargs) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index cd4360483..f5935c4ef 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -18,7 +18,7 @@ class Request(object_ref): def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None): + dont_filter=False, errback=None, flags=None, cb_kwargs=None): self._encoding = encoding # this one has to be set first self.method = str(method).upper() @@ -40,8 +40,15 @@ class Request(object_ref): self.dont_filter = dont_filter self._meta = dict(meta) if meta else None + self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) + @property + def cb_kwargs(self): + if self._cb_kwargs is None: + self._cb_kwargs = {} + return self._cb_kwargs + @property def meta(self): if self._meta is None: @@ -92,7 +99,7 @@ class Request(object_ref): given new values. """ for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback']: + 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 1974259b5..b0a526b72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -106,7 +106,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None): + dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -132,4 +132,5 @@ class Response(object_ref): encoding=encoding, priority=priority, dont_filter=dont_filter, - errback=errback) + errback=errback, + cb_kwargs=cb_kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 74a042f2c..339913d4e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -123,7 +123,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None): + dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -154,7 +154,8 @@ class TextResponse(Response): encoding=encoding, priority=priority, dont_filter=dont_filter, - errback=errback + errback=errback, + cb_kwargs=cb_kwargs, ) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 3c463cfed..c7ea7b425 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -32,7 +32,8 @@ def request_to_dict(request, spider=None): '_encoding': request._encoding, 'priority': request.priority, 'dont_filter': request.dont_filter, - 'flags': request.flags + 'flags': request.flags, + 'cb_kwargs': request.cb_kwargs, } if type(request) is not Request: d['_class'] = request.__module__ + '.' + request.__class__.__name__ @@ -64,7 +65,9 @@ def request_from_dict(d, spider=None): encoding=d['_encoding'], priority=d['priority'], dont_filter=d['dont_filter'], - flags=d.get('flags')) + flags=d.get('flags'), + cb_kwargs=d.get('cb_kwargs'), + ) def _is_private_method(name): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 02037b866..c18a6ce9f 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -43,6 +43,12 @@ class MySpider(scrapy.Spider): else: self.logger.debug('It Works!') + def parse_request_with_cb_kwargs(self, response, foo=None, key=None): + if foo == 'bar' and key == 'value': + self.logger.debug('It Works!') + else: + self.logger.debug('It Does Not Work :(') + def parse_request_without_meta(self, response): foo = response.meta.get('foo', 'bar') @@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_request_with_cb_kwargs(self): + raw_json_string = '{"foo" : "bar", "key": "value"}' + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--cbkwargs', raw_json_string, + '-c', 'parse_request_with_cb_kwargs', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_without_meta(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 49f148016..1bab37c0b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -181,6 +181,7 @@ class RequestTest(unittest.TestCase): r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], callback=somecallback, errback=somecallback) r1.meta['foo'] = 'bar' + r1.cb_kwargs['key'] = 'value' r2 = r1.copy() # make sure copy does not propagate callbacks @@ -193,6 +194,10 @@ class RequestTest(unittest.TestCase): assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" self.assertEqual(r1.flags, r2.flags) + # make sure cb_kwargs dict is shallow copied + assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical" + self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" self.assertEqual(r1.meta, r2.meta) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py new file mode 100644 index 000000000..c9943faa8 --- /dev/null +++ b/tests/test_request_cb_kwargs.py @@ -0,0 +1,169 @@ +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase +import six + +from scrapy.http import Request +from scrapy.crawler import CrawlerRunner +from tests.spiders import MockServerSpider +from tests.mockserver import MockServer + + +class InjectArgumentsDownloaderMiddleware(object): + """ + Make sure downloader middlewares are able to update the keyword arguments + """ + def process_request(self, request, spider): + if request.callback.__name__ == 'parse_downloader_mw': + request.cb_kwargs['from_process_request'] = True + return None + + def process_response(self, request, response, spider): + if request.callback.__name__ == 'parse_downloader_mw': + request.cb_kwargs['from_process_response'] = True + return response + + +class InjectArgumentsSpiderMiddleware(object): + """ + Make sure spider middlewares are able to update the keyword arguments + """ + def process_start_requests(self, start_requests, spider): + for request in start_requests: + if request.callback.__name__ == 'parse_spider_mw': + request.cb_kwargs['from_process_start_requests'] = True + yield request + + def process_spider_input(self, response, spider): + request = response.request + if request.callback.__name__ == 'parse_spider_mw': + request.cb_kwargs['from_process_spider_input'] = True + return None + + def process_spider_output(self, response, result, spider): + for element in result: + if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2': + element.cb_kwargs['from_process_spider_output'] = True + yield element + + +class KeywordArgumentsSpider(MockServerSpider): + name = 'kwargs' + custom_settings = { + 'DOWNLOADER_MIDDLEWARES': { + __name__ + '.InjectArgumentsDownloaderMiddleware': 750, + }, + 'SPIDER_MIDDLEWARES': { + __name__ + '.InjectArgumentsSpiderMiddleware': 750, + }, + } + + checks = list() + + def start_requests(self): + data = {'key': 'value', 'number': 123} + yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) + yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) + yield Request(self.mockserver.url('/general_without'), self.parse_general) + yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) + yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data) + yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data) + yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data) + yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw) + yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw) + + def parse_first(self, response, key, number): + self.checks.append(key == 'value') + self.checks.append(number == 123) + self.crawler.stats.inc_value('boolean_checks', 2) + yield response.follow( + self.mockserver.url('/two'), + self.parse_second, + cb_kwargs={'new_key': 'new_value'}) + + def parse_second(self, response, new_key): + self.checks.append(new_key == 'new_value') + self.crawler.stats.inc_value('boolean_checks') + + def parse_general(self, response, **kwargs): + if response.url.endswith('/general_with'): + self.checks.append(kwargs['key'] == 'value') + self.checks.append(kwargs['number'] == 123) + self.crawler.stats.inc_value('boolean_checks', 2) + elif response.url.endswith('/general_without'): + self.checks.append(kwargs == {}) + self.crawler.stats.inc_value('boolean_checks') + + def parse_no_kwargs(self, response): + self.checks.append(response.url.endswith('/no_kwargs')) + self.crawler.stats.inc_value('boolean_checks') + + def parse_default(self, response, key, number=None, default=99): + self.checks.append(response.url.endswith('/default')) + self.checks.append(key == 'value') + self.checks.append(number == 123) + self.checks.append(default == 99) + self.crawler.stats.inc_value('boolean_checks', 4) + + def parse_takes_less(self, response, key): + """ + Should raise + TypeError: parse_takes_less() got an unexpected keyword argument 'number' + """ + + def parse_takes_more(self, response, key, number, other): + """ + Should raise + TypeError: parse_takes_more() missing 1 required positional argument: 'other' + """ + + def parse_downloader_mw(self, response, from_process_request, from_process_response): + self.checks.append(bool(from_process_request)) + self.checks.append(bool(from_process_response)) + self.crawler.stats.inc_value('boolean_checks', 2) + + def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests): + self.checks.append(bool(from_process_spider_input)) + self.checks.append(bool(from_process_start_requests)) + self.crawler.stats.inc_value('boolean_checks', 2) + return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2) + + def parse_spider_mw_2(self, response, from_process_spider_output): + self.checks.append(bool(from_process_spider_output)) + self.crawler.stats.inc_value('boolean_checks', 1) + + +class CallbackKeywordArgumentsTestCase(TestCase): + + maxDiff = None + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_callback_kwargs(self): + crawler = self.runner.create_crawler(KeywordArgumentsSpider) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + self.assertTrue(all(crawler.spider.checks)) + self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks')) + # check exceptions for argument mismatch + exceptions = {} + for line in log.records: + for key in ('takes_less', 'takes_more'): + if key in line.getMessage(): + exceptions[key] = line + self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) + self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") + self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) + # py2 and py3 messages are different + exc_message = str(exceptions['takes_more'].exc_info[1]) + if six.PY2: + self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)") + elif six.PY3: + self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'") diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index e5a09dcf1..11ac56897 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -29,6 +29,7 @@ class RequestSerializationTest(unittest.TestCase): encoding='latin-1', priority=20, meta={'a': 'b'}, + cb_kwargs={'k': 'v'}, flags=['testFlag']) self._assert_serializes_ok(r, spider=self.spider) @@ -55,6 +56,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) + self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter)