From 6760bca74b1f51ce83ed73318d5ddcef03c9d129 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 Mar 2019 22:32:45 +0000 Subject: [PATCH] Rename Request.kwargs to Request.cb_kwargs --- docs/topics/request-response.rst | 16 ++++++++-------- scrapy/core/scraper.py | 4 +++- scrapy/http/request/__init__.py | 14 +++++++------- scrapy/http/response/__init__.py | 4 ++-- scrapy/http/response/text.py | 4 ++-- scrapy/utils/reqser.py | 4 ++-- tests/spiders.py | 6 +++--- tests/test_http_request.py | 8 ++++---- tests/test_utils_reqser.py | 4 ++-- 9 files changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d12766676..b3f849540 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs]) +.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -126,8 +126,8 @@ Request objects :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list - :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback. - :type kwargs: dict + :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback. + :type cb_kwargs: dict .. attribute:: Request.url @@ -168,7 +168,7 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. - .. attribute:: Request.kwargs + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents will be passed to the Request's callback as keyword arguments. It is empty @@ -177,7 +177,7 @@ Request objects This dict is `shallow copied`_ when the request is cloned using the ``copy()`` or ``replace()`` methods, and can also be accessed, in your - spider, from the ``response.kwargs`` attribute. + spider, from the ``response.cb_kwargs`` attribute. .. _shallow copied: https://docs.python.org/2/library/copy.html @@ -216,7 +216,7 @@ Example:: In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. The following two examples show how to achieve this by using the -:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively:: +:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively:: def parse_page1(self, response): item = MyItem() @@ -238,8 +238,8 @@ The following two examples show how to achieve this by using the item['main_url'] = response.url request = scrapy.Request("http://www.example.com/some_page.html", callback=self.parse_page2) - request.kwargs['item'] = item - request.kwargs['foo'] = 'bar' + request.cb_kwargs['item'] = item + request.cb_kwargs['foo'] = 'bar' yield request def parse_page2(self, response, item, foo): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7981ce231..08dd1acc5 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -143,7 +143,9 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs) + dfd.addCallbacks(callback=request.callback or spider.parse, + errback=request.errback, + callbackKeywords=request.cb_kwargs) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index c016eb727..f5935c4ef 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -18,7 +18,7 @@ class Request(object_ref): def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, kwargs=None): + dont_filter=False, errback=None, flags=None, cb_kwargs=None): self._encoding = encoding # this one has to be set first self.method = str(method).upper() @@ -40,14 +40,14 @@ class Request(object_ref): self.dont_filter = dont_filter self._meta = dict(meta) if meta else None - self._kwargs = dict(kwargs) if kwargs else None + self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) @property - def kwargs(self): - if self._kwargs is None: - self._kwargs = {} - return self._kwargs + def cb_kwargs(self): + if self._cb_kwargs is None: + self._cb_kwargs = {} + return self._cb_kwargs @property def meta(self): @@ -99,7 +99,7 @@ class Request(object_ref): given new values. """ for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']: + 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 99b04a26e..b0a526b72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -106,7 +106,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -133,4 +133,4 @@ class Response(object_ref): priority=priority, dont_filter=dont_filter, errback=errback, - kwargs=kwargs) + cb_kwargs=cb_kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 2039621b3..339913d4e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -123,7 +123,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -155,7 +155,7 @@ class TextResponse(Response): priority=priority, dont_filter=dont_filter, errback=errback, - kwargs=kwargs, + cb_kwargs=cb_kwargs, ) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d537057b1..e7016b92a 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -33,7 +33,7 @@ def request_to_dict(request, spider=None): 'priority': request.priority, 'dont_filter': request.dont_filter, 'flags': request.flags, - 'kwargs': request.kwargs, + 'cb_kwargs': request.cb_kwargs, } if type(request) is not Request: d['_class'] = request.__module__ + '.' + request.__class__.__name__ @@ -66,7 +66,7 @@ def request_from_dict(d, spider=None): priority=d['priority'], dont_filter=d['dont_filter'], flags=d.get('flags'), - kwargs=d.get('kwargs'), + cb_kwargs=d.get('cb_kwargs'), ) diff --git a/tests/spiders.py b/tests/spiders.py index 7b4707f62..a06985837 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -35,8 +35,8 @@ class KeywordArgumentsSpider(MockServerSpider): def start_requests(self): data = {'key': 'value', 'number': 123} - yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data) - yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data) + yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) + yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) @@ -47,7 +47,7 @@ class KeywordArgumentsSpider(MockServerSpider): yield response.follow( self.mockserver.url('/two'), self.parse_second, - kwargs={'new_key': 'new_value'}) + cb_kwargs={'new_key': 'new_value'}) def parse_second(self, response, new_key): self.checks.append(new_key == 'new_value') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 610893d8a..c1949a28c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -177,7 +177,7 @@ class RequestTest(unittest.TestCase): r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], callback=somecallback, errback=somecallback) r1.meta['foo'] = 'bar' - r1.kwargs['key'] = 'value' + r1.cb_kwargs['key'] = 'value' r2 = r1.copy() # make sure copy does not propagate callbacks @@ -190,9 +190,9 @@ class RequestTest(unittest.TestCase): assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" self.assertEqual(r1.flags, r2.flags) - # make sure kwargs dict is shallow copied - assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical" - self.assertEqual(r1.kwargs, r2.kwargs) + # make sure cb_kwargs dict is shallow copied + assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical" + self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 76de20f22..e1601b76b 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -26,7 +26,7 @@ class RequestSerializationTest(unittest.TestCase): encoding='latin-1', priority=20, meta={'a': 'b'}, - kwargs={'k': 'v'}, + cb_kwargs={'k': 'v'}, flags=['testFlag']) self._assert_serializes_ok(r, spider=self.spider) @@ -53,7 +53,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) - self.assertEqual(r1.kwargs, r2.kwargs) + self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter)