From 50a0d87d1e472fcc514f3dc2b028b653b7826a9c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:20:08 -0300
Subject: [PATCH 01/21] Passing keyword arguments to callbacks

---
 scrapy/core/scraper.py          | 2 +-
 scrapy/http/request/__init__.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index ee1e95a0c..7981ce231 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -143,7 +143,7 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        dfd.addCallbacks(request.callback or spider.parse, request.errback)
+        dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, response, spider):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index cd4360483..7d5cc9dae 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -18,7 +18,7 @@ class Request(object_ref):
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None, flags=None):
+                 dont_filter=False, errback=None, flags=None, kwargs=None):
 
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
@@ -41,6 +41,7 @@ class Request(object_ref):
 
         self._meta = dict(meta) if meta else None
         self.flags = [] if flags is None else list(flags)
+        self.kwargs = dict(kwargs) if kwargs else None
 
     @property
     def meta(self):

From a2b509a42266a2ab3389de64b608e616f88f77e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:38:06 -0300
Subject: [PATCH 02/21] Pass callback kwargs with response.follow

---
 scrapy/http/response/__init__.py | 5 +++--
 scrapy/http/response/text.py     | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 1974259b5..99b04a26e 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -106,7 +106,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None):
+               dont_filter=False, errback=None, kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -132,4 +132,5 @@ class Response(object_ref):
                        encoding=encoding,
                        priority=priority,
                        dont_filter=dont_filter,
-                       errback=errback)
+                       errback=errback,
+                       kwargs=kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 74a042f2c..2039621b3 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -123,7 +123,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None):
+               dont_filter=False, errback=None, kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -154,7 +154,8 @@ class TextResponse(Response):
             encoding=encoding,
             priority=priority,
             dont_filter=dont_filter,
-            errback=errback
+            errback=errback,
+            kwargs=kwargs,
         )
 
 

From 69a1ee79aa43bb1444e5b2a800a2e1702db6866e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:38:29 -0300
Subject: [PATCH 03/21] Copy request.kwargs

---
 scrapy/http/request/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 7d5cc9dae..9a155f415 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -93,7 +93,7 @@ class Request(object_ref):
         given new values.
         """
         for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
-                  'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
+                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)

From a67f1ce512ffa111a400e2718ebbf3e7bd32d0ae Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 3 Jan 2019 17:49:41 -0300
Subject: [PATCH 04/21] Serialize Request kwargs

---
 scrapy/utils/reqser.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 959dddbd5..d537057b1 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -32,7 +32,8 @@ def request_to_dict(request, spider=None):
         '_encoding': request._encoding,
         'priority': request.priority,
         'dont_filter': request.dont_filter,
-        'flags': request.flags
+        'flags': request.flags,
+        'kwargs': request.kwargs,
     }
     if type(request) is not Request:
         d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -64,7 +65,9 @@ def request_from_dict(d, spider=None):
         encoding=d['_encoding'],
         priority=d['priority'],
         dont_filter=d['dont_filter'],
-        flags=d.get('flags'))
+        flags=d.get('flags'),
+        kwargs=d.get('kwargs'),
+    )
 
 
 def _find_method(obj, func):

From 770a501fb32b7582acaa5900ac2f41ea46a321cd Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 9 Jan 2019 10:40:03 -0300
Subject: [PATCH 05/21] Test request kwargs (copy, serialization)

---
 scrapy/http/request/__init__.py | 8 +++++++-
 tests/test_http_request.py      | 5 +++++
 tests/test_utils_reqser.py      | 2 ++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 9a155f415..c016eb727 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -40,8 +40,14 @@ class Request(object_ref):
         self.dont_filter = dont_filter
 
         self._meta = dict(meta) if meta else None
+        self._kwargs = dict(kwargs) if kwargs else None
         self.flags = [] if flags is None else list(flags)
-        self.kwargs = dict(kwargs) if kwargs else None
+
+    @property
+    def kwargs(self):
+        if self._kwargs is None:
+            self._kwargs = {}
+        return self._kwargs
 
     @property
     def meta(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 58326a384..610893d8a 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -177,6 +177,7 @@ class RequestTest(unittest.TestCase):
         r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
                                 callback=somecallback, errback=somecallback)
         r1.meta['foo'] = 'bar'
+        r1.kwargs['key'] = 'value'
         r2 = r1.copy()
 
         # make sure copy does not propagate callbacks
@@ -189,6 +190,10 @@ class RequestTest(unittest.TestCase):
         assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
+        # make sure kwargs dict is shallow copied
+        assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical"
+        self.assertEqual(r1.kwargs, r2.kwargs)
+
         # make sure meta dict is shallow copied
         assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
         self.assertEqual(r1.meta, r2.meta)
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index dcc070b8f..76de20f22 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -26,6 +26,7 @@ class RequestSerializationTest(unittest.TestCase):
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
+            kwargs={'k': 'v'},
             flags=['testFlag'])
         self._assert_serializes_ok(r, spider=self.spider)
 
@@ -52,6 +53,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.headers, r2.headers)
         self.assertEqual(r1.cookies, r2.cookies)
         self.assertEqual(r1.meta, r2.meta)
+        self.assertEqual(r1.kwargs, r2.kwargs)
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)

From 57e7c769779b9d37058d1d8839215a9c269b8c5b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 9 Jan 2019 10:40:44 -0300
Subject: [PATCH 06/21] Test callback kwargs

---
 tests/spiders.py    | 34 ++++++++++++++++++++++++++++++++++
 tests/test_crawl.py |  8 +++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 7816bf7c7..5a1471072 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -28,6 +28,40 @@ class MetaSpider(MockServerSpider):
         self.meta['close_reason'] = reason
 
 
+class KeywordArgumentsSpider(MockServerSpider):
+
+    name = 'kwargs'
+    checks = set()
+
+    def start_requests(self):
+        data = {'key': 'value', 'number': 123}
+        yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data)
+        yield Request(self.mockserver.url('/general_without'), self.parse_general)
+        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+
+    def parse_first(self, response, key, number):
+        self.checks.add(key == 'value')
+        self.checks.add(number == 123)
+        yield response.follow(
+            self.mockserver.url('/two'),
+            self.parse_second,
+            kwargs={'new_key': 'new_value'})
+
+    def parse_second(self, response, new_key):
+        self.checks.add(new_key == 'new_value')
+
+    def parse_general(self, response, **kwargs):
+        if response.url.endswith('/general_with'):
+            self.checks.add(kwargs['key'] == 'value')
+            self.checks.add(kwargs['number'] == 123)
+        elif response.url.endswith('/general_without'):
+            self.checks.add(kwargs == {})
+
+    def parse_no_kwargs(self, response):
+        pass
+
+
 class FollowAllSpider(MetaSpider):
 
     name = 'follow'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..3879a017c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -8,7 +8,7 @@ from twisted.trial.unittest import TestCase
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
 from scrapy.utils.python import to_unicode
-from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
+from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, KeywordArgumentsSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
 
@@ -23,6 +23,12 @@ class CrawlTestCase(TestCase):
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
+    @defer.inlineCallbacks
+    def test_callback_kwargs(self):
+        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(crawler.spider.checks, set([True]))
+
     @defer.inlineCallbacks
     def test_follow_all(self):
         crawler = self.runner.create_crawler(FollowAllSpider)

From bddfeaba4c17040b2986403f8b2ba25d4252e1b5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 15 Jan 2019 15:35:46 -0300
Subject: [PATCH 07/21] Add Request.kwargs docs

---
 docs/topics/request-response.rst | 40 +++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e29914dbf..d12766676 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs])
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -126,6 +126,9 @@ Request objects
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
 
+    :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+    :type kwargs: dict
+
     .. attribute:: Request.url
 
         A string containing the URL of this request. Keep in mind that this
@@ -165,6 +168,17 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
+    .. attribute:: Request.kwargs
+
+        A dictionary that contains arbitrary metadata for this request. Its contents
+        will be passed to the Request's callback as keyword arguments. It is empty
+        for new Requests, which means by default callbacks only get a :class:`Response`
+        object as argument.
+
+        This dict is `shallow copied`_ when the request is cloned using the
+        ``copy()`` or ``replace()`` methods, and can also be accessed, in your
+        spider, from the ``response.kwargs`` attribute.
+
     .. _shallow copied: https://docs.python.org/2/library/copy.html
 
     .. method:: Request.copy()
@@ -200,11 +214,9 @@ Example::
         self.logger.info("Visited %s", response.url)
 
 In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback. You
-can use the :attr:`Request.meta` attribute for that.
-
-Here's an example of how to pass an item using this mechanism, to populate
-different fields from different pages::
+functions so you can receive the arguments later, in the second callback.
+The following two examples show how to achieve this by using the 
+:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively::
 
     def parse_page1(self, response):
         item = MyItem()
@@ -219,6 +231,22 @@ different fields from different pages::
         item['other_url'] = response.url
         yield item
 
+::
+
+    def parse_page1(self, response):
+        item = MyItem()
+        item['main_url'] = response.url
+        request = scrapy.Request("http://www.example.com/some_page.html",
+                                 callback=self.parse_page2)
+        request.kwargs['item'] = item
+        request.kwargs['foo'] = 'bar'
+        yield request
+
+    def parse_page2(self, response, item, foo):
+        item['other_url'] = response.url
+        item['foo'] = foo
+        yield item
+
 
 .. _topics-request-response-ref-errbacks:
 

From 645e8d16a4c966b50bd39667aaef28dc1eeb43b8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 Mar 2019 22:20:36 +0000
Subject: [PATCH 08/21] Count keyword argument checks

---
 tests/spiders.py    | 21 +++++++++++++--------
 tests/test_crawl.py |  3 ++-
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 5a1471072..7b4707f62 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -31,7 +31,7 @@ class MetaSpider(MockServerSpider):
 class KeywordArgumentsSpider(MockServerSpider):
 
     name = 'kwargs'
-    checks = set()
+    checks = list()
 
     def start_requests(self):
         data = {'key': 'value', 'number': 123}
@@ -41,25 +41,30 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
 
     def parse_first(self, response, key, number):
-        self.checks.add(key == 'value')
-        self.checks.add(number == 123)
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.crawler.stats.inc_value('boolean_checks', 2)
         yield response.follow(
             self.mockserver.url('/two'),
             self.parse_second,
             kwargs={'new_key': 'new_value'})
 
     def parse_second(self, response, new_key):
-        self.checks.add(new_key == 'new_value')
+        self.checks.append(new_key == 'new_value')
+        self.crawler.stats.inc_value('boolean_checks')
 
     def parse_general(self, response, **kwargs):
         if response.url.endswith('/general_with'):
-            self.checks.add(kwargs['key'] == 'value')
-            self.checks.add(kwargs['number'] == 123)
+            self.checks.append(kwargs['key'] == 'value')
+            self.checks.append(kwargs['number'] == 123)
+            self.crawler.stats.inc_value('boolean_checks', 2)
         elif response.url.endswith('/general_without'):
-            self.checks.add(kwargs == {})
+            self.checks.append(kwargs == {})
+            self.crawler.stats.inc_value('boolean_checks')
 
     def parse_no_kwargs(self, response):
-        pass
+        self.checks.append(response.url.endswith('/no_kwargs'))
+        self.crawler.stats.inc_value('boolean_checks')
 
 
 class FollowAllSpider(MetaSpider):
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3879a017c..9a39b8cb4 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -27,7 +27,8 @@ class CrawlTestCase(TestCase):
     def test_callback_kwargs(self):
         crawler = self.runner.create_crawler(KeywordArgumentsSpider)
         yield crawler.crawl(mockserver=self.mockserver)
-        self.assertEqual(crawler.spider.checks, set([True]))
+        self.assertTrue(all(crawler.spider.checks))
+        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
 
     @defer.inlineCallbacks
     def test_follow_all(self):

From 6760bca74b1f51ce83ed73318d5ddcef03c9d129 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 Mar 2019 22:32:45 +0000
Subject: [PATCH 09/21] Rename Request.kwargs to Request.cb_kwargs

---
 docs/topics/request-response.rst | 16 ++++++++--------
 scrapy/core/scraper.py           |  4 +++-
 scrapy/http/request/__init__.py  | 14 +++++++-------
 scrapy/http/response/__init__.py |  4 ++--
 scrapy/http/response/text.py     |  4 ++--
 scrapy/utils/reqser.py           |  4 ++--
 tests/spiders.py                 |  6 +++---
 tests/test_http_request.py       |  8 ++++----
 tests/test_utils_reqser.py       |  4 ++--
 9 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d12766676..b3f849540 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -126,8 +126,8 @@ Request objects
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
 
-    :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
-    :type kwargs: dict
+    :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+    :type cb_kwargs: dict
 
     .. attribute:: Request.url
 
@@ -168,7 +168,7 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
-    .. attribute:: Request.kwargs
+    .. attribute:: Request.cb_kwargs
 
         A dictionary that contains arbitrary metadata for this request. Its contents
         will be passed to the Request's callback as keyword arguments. It is empty
@@ -177,7 +177,7 @@ Request objects
 
         This dict is `shallow copied`_ when the request is cloned using the
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
-        spider, from the ``response.kwargs`` attribute.
+        spider, from the ``response.cb_kwargs`` attribute.
 
     .. _shallow copied: https://docs.python.org/2/library/copy.html
 
@@ -216,7 +216,7 @@ Example::
 In some cases you may be interested in passing arguments to those callback
 functions so you can receive the arguments later, in the second callback.
 The following two examples show how to achieve this by using the 
-:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively::
+:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively::
 
     def parse_page1(self, response):
         item = MyItem()
@@ -238,8 +238,8 @@ The following two examples show how to achieve this by using the
         item['main_url'] = response.url
         request = scrapy.Request("http://www.example.com/some_page.html",
                                  callback=self.parse_page2)
-        request.kwargs['item'] = item
-        request.kwargs['foo'] = 'bar'
+        request.cb_kwargs['item'] = item
+        request.cb_kwargs['foo'] = 'bar'
         yield request
 
     def parse_page2(self, response, item, foo):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 7981ce231..08dd1acc5 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -143,7 +143,9 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs)
+        dfd.addCallbacks(callback=request.callback or spider.parse,
+                         errback=request.errback,
+                         callbackKeywords=request.cb_kwargs)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, response, spider):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index c016eb727..f5935c4ef 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -18,7 +18,7 @@ class Request(object_ref):
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None, flags=None, kwargs=None):
+                 dont_filter=False, errback=None, flags=None, cb_kwargs=None):
 
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
@@ -40,14 +40,14 @@ class Request(object_ref):
         self.dont_filter = dont_filter
 
         self._meta = dict(meta) if meta else None
-        self._kwargs = dict(kwargs) if kwargs else None
+        self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
         self.flags = [] if flags is None else list(flags)
 
     @property
-    def kwargs(self):
-        if self._kwargs is None:
-            self._kwargs = {}
-        return self._kwargs
+    def cb_kwargs(self):
+        if self._cb_kwargs is None:
+            self._cb_kwargs = {}
+        return self._cb_kwargs
 
     @property
     def meta(self):
@@ -99,7 +99,7 @@ class Request(object_ref):
         given new values.
         """
         for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
-                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']:
+                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 99b04a26e..b0a526b72 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -106,7 +106,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -133,4 +133,4 @@ class Response(object_ref):
                        priority=priority,
                        dont_filter=dont_filter,
                        errback=errback,
-                       kwargs=kwargs)
+                       cb_kwargs=cb_kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 2039621b3..339913d4e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -123,7 +123,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -155,7 +155,7 @@ class TextResponse(Response):
             priority=priority,
             dont_filter=dont_filter,
             errback=errback,
-            kwargs=kwargs,
+            cb_kwargs=cb_kwargs,
         )
 
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d537057b1..e7016b92a 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -33,7 +33,7 @@ def request_to_dict(request, spider=None):
         'priority': request.priority,
         'dont_filter': request.dont_filter,
         'flags': request.flags,
-        'kwargs': request.kwargs,
+        'cb_kwargs': request.cb_kwargs,
     }
     if type(request) is not Request:
         d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -66,7 +66,7 @@ def request_from_dict(d, spider=None):
         priority=d['priority'],
         dont_filter=d['dont_filter'],
         flags=d.get('flags'),
-        kwargs=d.get('kwargs'),
+        cb_kwargs=d.get('cb_kwargs'),
     )
 
 
diff --git a/tests/spiders.py b/tests/spiders.py
index 7b4707f62..a06985837 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -35,8 +35,8 @@ class KeywordArgumentsSpider(MockServerSpider):
 
     def start_requests(self):
         data = {'key': 'value', 'number': 123}
-        yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data)
-        yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data)
+        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_without'), self.parse_general)
         yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
 
@@ -47,7 +47,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield response.follow(
             self.mockserver.url('/two'),
             self.parse_second,
-            kwargs={'new_key': 'new_value'})
+            cb_kwargs={'new_key': 'new_value'})
 
     def parse_second(self, response, new_key):
         self.checks.append(new_key == 'new_value')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 610893d8a..c1949a28c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -177,7 +177,7 @@ class RequestTest(unittest.TestCase):
         r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
                                 callback=somecallback, errback=somecallback)
         r1.meta['foo'] = 'bar'
-        r1.kwargs['key'] = 'value'
+        r1.cb_kwargs['key'] = 'value'
         r2 = r1.copy()
 
         # make sure copy does not propagate callbacks
@@ -190,9 +190,9 @@ class RequestTest(unittest.TestCase):
         assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
-        # make sure kwargs dict is shallow copied
-        assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical"
-        self.assertEqual(r1.kwargs, r2.kwargs)
+        # make sure cb_kwargs dict is shallow copied
+        assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical"
+        self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
 
         # make sure meta dict is shallow copied
         assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 76de20f22..e1601b76b 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -26,7 +26,7 @@ class RequestSerializationTest(unittest.TestCase):
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
-            kwargs={'k': 'v'},
+            cb_kwargs={'k': 'v'},
             flags=['testFlag'])
         self._assert_serializes_ok(r, spider=self.spider)
 
@@ -53,7 +53,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.headers, r2.headers)
         self.assertEqual(r1.cookies, r2.cookies)
         self.assertEqual(r1.meta, r2.meta)
-        self.assertEqual(r1.kwargs, r2.kwargs)
+        self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)

From 8528f5065f99046b149b5e1901d6cbe5296f048a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 27 Mar 2019 14:42:26 -0300
Subject: [PATCH 10/21] [Doc] Update cb_kwargs example

---
 docs/topics/request-response.rst | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b3f849540..61789be0f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -233,19 +233,19 @@ The following two examples show how to achieve this by using the
 
 ::
 
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.cb_kwargs['item'] = item
-        request.cb_kwargs['foo'] = 'bar'
+    def parse(self, response):
+        request = scrapy.Request('http://www.example.com/index.html',
+                                 callback=self.parse_page2,
+                                 cb_kwargs=dict(main_url=response.url))
+        request.cb_kwargs['foo'] = 'bar'  # add more arguments for the callback
         yield request
 
-    def parse_page2(self, response, item, foo):
-        item['other_url'] = response.url
-        item['foo'] = foo
-        yield item
+    def parse_page2(self, response, main_url, foo):
+        yield dict(
+            main_url=main_url,
+            other_url=response.url,
+            foo=foo,
+        )
 
 
 .. _topics-request-response-ref-errbacks:

From 70a4d93aa324fb276e60d641b37bdc6eb707b1cb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 10:40:41 -0300
Subject: [PATCH 11/21] Callback kwargs: more tests

---
 tests/spiders.py    | 22 ++++++++++++++++++++++
 tests/test_crawl.py | 21 ++++++++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index a06985837..8c8d50ff5 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -39,6 +39,9 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_without'), self.parse_general)
         yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
 
     def parse_first(self, response, key, number):
         self.checks.append(key == 'value')
@@ -66,6 +69,25 @@ class KeywordArgumentsSpider(MockServerSpider):
         self.checks.append(response.url.endswith('/no_kwargs'))
         self.crawler.stats.inc_value('boolean_checks')
 
+    def parse_default(self, response, key, number=None, default=99):
+        self.checks.append(response.url.endswith('/default'))
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.checks.append(default == 99)
+        self.crawler.stats.inc_value('boolean_checks', 4)
+
+    def parse_takes_less(self, response, key):
+        """
+        Should raise
+        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
+        """
+
+    def parse_takes_more(self, response, key, number, other):
+        """
+        Should raise
+        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
+        """
+
 
 class FollowAllSpider(MetaSpider):
 
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 9a39b8cb4..2b3e56ee9 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -4,6 +4,7 @@ import logging
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
+import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -15,6 +16,8 @@ from tests.mockserver import MockServer
 
 class CrawlTestCase(TestCase):
 
+    maxDiff = None
+
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -26,9 +29,25 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_callback_kwargs(self):
         crawler = self.runner.create_crawler(KeywordArgumentsSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(all(crawler.spider.checks))
         self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+        # check exceptions for argument mismatch
+        exceptions = {}
+        for line in log.records:
+            for key in ('takes_less', 'takes_more'):
+                if key in line.getMessage():
+                    exceptions[key] = line
+        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+        # py2 and py3 messages are different
+        exc_message = str(exceptions['takes_more'].exc_info[1])
+        if six.PY2:
+            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
+        elif six.PY3:
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
 
     @defer.inlineCallbacks
     def test_follow_all(self):

From 3efe3bea1cbb5ae83c024fc6dc8e1776a47a345f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 14:16:03 -0300
Subject: [PATCH 12/21] Update docs about cb_kwargs and meta

---
 docs/topics/request-response.rst | 41 ++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index dd0db8156..05ca8d6c1 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -215,24 +215,12 @@ Example::
 
 In some cases you may be interested in passing arguments to those callback
 functions so you can receive the arguments later, in the second callback.
-The following two examples show how to achieve this by using the 
-:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively::
-
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.meta['item'] = item
-        yield request
-
-    def parse_page2(self, response):
-        item = response.meta['item']
-        item['other_url'] = response.url
-        yield item
+The following example shows how to achieve this by using the
+:attr:`Request.cb_kwargs` attribute:
 
 ::
 
+    # pass information to the next callback using the Request.cb_kwargs attribute
     def parse(self, response):
         request = scrapy.Request('http://www.example.com/index.html',
                                  callback=self.parse_page2,
@@ -247,6 +235,29 @@ The following two examples show how to achieve this by using the
             foo=foo,
         )
 
+.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
+   Prior to that, :attr:`Request.meta` was the recommended option for passing
+   information around callbacks. However, after ``1.7`` :attr:`Request.cb_kwargs`
+   became the preferred way of passing user information, leaving :attr:`Request.meta`
+   to be used by internal components like spider or downloader middlewares.
+   The following example, which uses :attr:`Request.meta`, is only kept for historical
+   reasons.
+
+::
+
+    # pass information to the next callback using the Request.meta attribute
+    def parse_page1(self, response):
+        item = MyItem()
+        item['main_url'] = response.url
+        request = scrapy.Request("http://www.example.com/some_page.html",
+                                 callback=self.parse_page2)
+        request.meta['item'] = item
+        yield request
+
+    def parse_page2(self, response):
+        item = response.meta['item']
+        item['other_url'] = response.url
+        yield item
 
 .. _topics-request-response-ref-errbacks:
 

From e8af6331b5ff62d71ff80eddcc52b85c25482c0e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 14:56:31 -0300
Subject: [PATCH 13/21] Add cb_kwargs option to the parse command

---
 docs/topics/commands.rst    |  3 +++
 scrapy/commands/parse.py    | 37 ++++++++++++++++++++++++-------------
 tests/test_command_parse.py | 14 ++++++++++++++
 3 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 97f8311de..6644d65e4 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -461,6 +461,9 @@ Supported options:
 * ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
+* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback.
+  This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}'
+
 * ``--pipelines``: process items through pipelines
 
 * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 69418a478..2486f3f23 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,12 +51,13 @@ class Command(ScrapyCommand):
             help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
+        parser.add_option("--cb_kwargs", dest="cb_kwargs",
+            help="inject extra cb_kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
             help="print each depth level one by one")
 
-
     @property
     def max_level(self):
         levels = list(self.items.keys()) + list(self.requests.keys())
@@ -111,10 +112,11 @@ class Command(ScrapyCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def run_callback(self, response, cb):
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
         items, requests = [], []
 
-        for x in iterate_spider_output(cb(response)):
+        for x in iterate_spider_output(callback(response, **cb_kwargs)):
             if isinstance(x, (BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
@@ -142,8 +144,7 @@ class Command(ScrapyCommand):
         else:
             self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
-                logger.error('Unable to find spider for: %(url)s',
-                             {'url': url})
+                logger.error('Unable to find spider for: %(url)s', {'url': url})
 
         # Request requires callback argument as callable or None, not string
         request = Request(url, None)
@@ -160,7 +161,7 @@ class Command(ScrapyCommand):
                          {'url': url})
 
     def prepare_request(self, spider, request, opts):
-        def callback(response):
+        def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
@@ -175,7 +176,7 @@ class Command(ScrapyCommand):
 
                     if not cb:
                         logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
-                                 {'url': response.url, 'spider': spider.name})
+                                     {'url': response.url, 'spider': spider.name})
                         return
                 else:
                     cb = 'parse'
@@ -192,7 +193,7 @@ class Command(ScrapyCommand):
             # parse items and requests
             depth = response.meta['_depth']
 
-            items, requests = self.run_callback(response, cb)
+            items, requests = self.run_callback(response, cb, cb_kwargs)
             if opts.pipelines:
                 itemproc = self.pcrawler.engine.scraper.itemproc
                 for item in items:
@@ -207,10 +208,14 @@ class Command(ScrapyCommand):
                     req.callback = callback
                 return requests
 
-        #update request meta if any extra meta was passed through the --meta/-m opts.
+        # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
             request.meta.update(opts.meta)
 
+        # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option.
+        if opts.cb_kwargs:
+            request.cb_kwargs.update(opts.cb_kwargs)
+
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
         request.callback = callback
@@ -221,23 +226,29 @@ class Command(ScrapyCommand):
 
         self.process_spider_arguments(opts)
         self.process_request_meta(opts)
+        self.process_request_cb_kwargs(opts)
 
     def process_spider_arguments(self, opts):
-
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
 
     def process_request_meta(self, opts):
-
         if opts.meta:
             try:
                 opts.meta = json.loads(opts.meta)
             except ValueError:
-                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
-                                "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+                                 "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
 
+    def process_request_cb_kwargs(self, opts):
+        if opts.cb_kwargs:
+            try:
+                opts.cb_kwargs = json.loads(opts.cb_kwargs)
+            except ValueError:
+                raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. "
+                                 "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False)
 
     def run(self, args, opts):
         # parse arguments
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 02037b866..1404005fb 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -43,6 +43,12 @@ class MySpider(scrapy.Spider):
         else:
             self.logger.debug('It Works!')
 
+    def parse_request_with_cb_kwargs(self, response, foo=None, key=None):
+        if foo == 'bar' and key == 'value':
+            self.logger.debug('It Works!')
+        else:
+            self.logger.debug('It Does Not Work :(')
+
     def parse_request_without_meta(self, response):
         foo = response.meta.get('foo', 'bar')
 
@@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_request_with_cb_kwargs(self):
+        raw_json_string = '{"foo" : "bar", "key": "value"}'
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '--cb_kwargs', raw_json_string,
+                                           '-c', 'parse_request_with_cb_kwargs',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):

From 8fb077694fcaa50a8625c8e2e8d0068add2b056d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 28 Mar 2019 15:18:00 -0300
Subject: [PATCH 14/21] Request.cb_kwargs: Update docs

---
 docs/topics/debug.rst            | 13 +++++--------
 docs/topics/jobs.rst             |  9 +++++----
 docs/topics/leaks.rst            | 14 ++++++++------
 docs/topics/request-response.rst | 13 ++++++-------
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index f93aa2c72..0aaad0c77 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -28,16 +28,15 @@ Consider the following scrapy spider below::
             item = MyItem()
             # populate `item` fields
             # and extract item_details_url
-            yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
+            yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
 
-        def parse_details(self, response):
-            item = response.meta['item']
+        def parse_details(self, response, item):
             # populate more `item` fields
             return item
 
 Basically this is a simple spider which parses two pages of items (the
 start_urls). Items also have a details page with additional information, so we
-use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
 partially populated item.
 
 
@@ -100,8 +99,7 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see
 
     from scrapy.shell import inspect_response
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item
@@ -134,8 +132,7 @@ Logging is another useful option for getting information about your spider run.
 Although not as convenient, it comes with the advantage that the logs will be
 available in all future runs should they be necessary again::
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 1a5d52487..9fd311c69 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -81,7 +81,8 @@ So, for example, this won't work::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
+        return scrapy.Request('http://www.example.com',
+                              callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
         print("the argument passed is: %s" % somearg)
@@ -90,10 +91,10 @@ But this will::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
+        return scrapy.Request('http://www.example.com',
+                              callback=self.other_callback, cb_kwargs={'somearg': somearg})
 
-    def other_callback(self, response):
-        somearg = response.meta['somearg']
+    def other_callback(self, response, somearg):
         print("the argument passed is: %s" % somearg)
 
 If you wish to log the requests that couldn't be serialized, you can set the
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index af14d14e8..8278e9849 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -27,10 +27,11 @@ Common causes of memory leaks
 
 It happens quite often (sometimes by accident, sometimes on purpose) that the
 Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
-and that effectively bounds the lifetime of those referenced objects to the
-lifetime of the Request. This is, by far, the most common cause of memory leaks
-in Scrapy projects, and a quite difficult one to debug for newcomers.
+:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+attributes or the request callback function) and that effectively bounds the
+lifetime of those referenced objects to the lifetime of the Request. This is,
+by far, the most common cause of memory leaks in Scrapy projects, and a quite
+difficult one to debug for newcomers.
 
 In big projects, the spiders are typically written by different people and some
 of those spiders could be "leaking" and thus affecting the rest of the other
@@ -48,7 +49,8 @@ Too Many Requests?
 
 By default Scrapy keeps the request queue in memory; it includes
 :class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
+and :attr:`~scrapy.http.Request.meta`).
 While not necessarily a leak, this can take a lot of memory. Enabling
 :ref:`persistent job queue <topics-jobs>` could help keeping memory usage
 in control.
@@ -101,7 +103,7 @@ Let's see a concrete example of a hypothetical case of memory leaks.
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
-        callback=self.parse, meta={referer: response})
+                   callback=self.parse, cb_kwargs={'referer': response})
 
 That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 05ca8d6c1..f299c2cff 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -186,12 +186,12 @@ Request objects
        Return a new Request which is a copy of this Request. See also:
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
-    .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
+    .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
-       attribute :attr:`Request.meta` is copied by default (unless a new value
-       is given in the ``meta`` argument). See also
+       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are copied by default
+       (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
 .. _topics-request-response-ref-request-callback-arguments:
@@ -237,11 +237,10 @@ The following example shows how to achieve this by using the
 
 .. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
    Prior to that, :attr:`Request.meta` was the recommended option for passing
-   information around callbacks. However, after ``1.7`` :attr:`Request.cb_kwargs`
+   information around callbacks. However, after ``1.7``, using :attr:`Request.cb_kwargs`
    became the preferred way of passing user information, leaving :attr:`Request.meta`
-   to be used by internal components like spider or downloader middlewares.
-   The following example, which uses :attr:`Request.meta`, is only kept for historical
-   reasons.
+   to be populated by internal components like spider or downloader middlewares.
+   The following :attr:`Request.meta` example is only kept for historical reasons.
 
 ::
 

From f5e0b6b89ace437af850e0225651329101a59862 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 14:03:26 -0300
Subject: [PATCH 15/21] parse command: rename cb_kwargs option to cbkwargs

---
 scrapy/commands/parse.py    | 18 +++++++++---------
 tests/test_command_parse.py |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2486f3f23..e948d6406 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,8 +51,8 @@ class Command(ScrapyCommand):
             help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
-        parser.add_option("--cb_kwargs", dest="cb_kwargs",
-            help="inject extra cb_kwargs into the Request, it must be a valid raw json string")
+        parser.add_option("--cbkwargs", dest="cbkwargs",
+            help="inject extra cbkwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
@@ -212,9 +212,9 @@ class Command(ScrapyCommand):
         if opts.meta:
             request.meta.update(opts.meta)
 
-        # update cb_kwargs if any extra cb_kwargs was passed through the --cb_kwargs option.
-        if opts.cb_kwargs:
-            request.cb_kwargs.update(opts.cb_kwargs)
+        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
+        if opts.cbkwargs:
+            request.cb_kwargs.update(opts.cbkwargs)
 
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
@@ -243,12 +243,12 @@ class Command(ScrapyCommand):
                                  "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
 
     def process_request_cb_kwargs(self, opts):
-        if opts.cb_kwargs:
+        if opts.cbkwargs:
             try:
-                opts.cb_kwargs = json.loads(opts.cb_kwargs)
+                opts.cbkwargs = json.loads(opts.cbkwargs)
             except ValueError:
-                raise UsageError("Invalid --cb_kwargs value, pass a valid json string to --cb_kwargs. "
-                                 "Example: --cb_kwargs='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+                                 "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
 
     def run(self, args, opts):
         # parse arguments
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 1404005fb..c18a6ce9f 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -130,7 +130,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_request_with_cb_kwargs(self):
         raw_json_string = '{"foo" : "bar", "key": "value"}'
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
-                                           '--cb_kwargs', raw_json_string,
+                                           '--cbkwargs', raw_json_string,
                                            '-c', 'parse_request_with_cb_kwargs',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))

From ccb56a317ee249978496e918ee5b74d83c2d7199 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 14:12:26 -0300
Subject: [PATCH 16/21] Update docs about cb_kwargs and meta

---
 docs/topics/request-response.rst | 30 ++++++------------------------
 1 file changed, 6 insertions(+), 24 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f299c2cff..4e81ce878 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -190,8 +190,8 @@ Request objects
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
-       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are copied by default
-       (unless new values are given as arguments). See also
+       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
+       copied by default (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
 .. _topics-request-response-ref-request-callback-arguments:
@@ -220,7 +220,6 @@ The following example shows how to achieve this by using the
 
 ::
 
-    # pass information to the next callback using the Request.cb_kwargs attribute
     def parse(self, response):
         request = scrapy.Request('http://www.example.com/index.html',
                                  callback=self.parse_page2,
@@ -236,27 +235,10 @@ The following example shows how to achieve this by using the
         )
 
 .. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
-   Prior to that, :attr:`Request.meta` was the recommended option for passing
-   information around callbacks. However, after ``1.7``, using :attr:`Request.cb_kwargs`
-   became the preferred way of passing user information, leaving :attr:`Request.meta`
-   to be populated by internal components like spider or downloader middlewares.
-   The following :attr:`Request.meta` example is only kept for historical reasons.
-
-::
-
-    # pass information to the next callback using the Request.meta attribute
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.meta['item'] = item
-        yield request
-
-    def parse_page2(self, response):
-        item = response.meta['item']
-        item['other_url'] = response.url
-        yield item
+   Prior to that, using :attr:`Request.meta` was recommended for passing
+   information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
+   became the preferred way for handling user information, leaving :attr:`Request.meta`
+   for communication with components like middlewares and extensions.
 
 .. _topics-request-response-ref-errbacks:
 

From 294ef51bb24782a0527892ea93bb4876daa7ca50 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 16:12:55 -0300
Subject: [PATCH 17/21] parse command: update docs about passing callback
 keyword arguments

---
 docs/topics/commands.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 6644d65e4..a93bee06b 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -461,8 +461,8 @@ Supported options:
 * ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
-* ``--cb_kwargs``: additional keyword arguments that will be passed to the callback.
-  This must be a valid json string. Example: --cb_kwargs='{"foo" : "bar"}'
+* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
+  This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
 
 * ``--pipelines``: process items through pipelines
 

From 0522fe35c334141e90741644fec368cdbd12044e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 29 Mar 2019 16:15:34 -0300
Subject: [PATCH 18/21] parse command: improve option description

---
 scrapy/commands/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index e948d6406..d4f2234b0 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -52,7 +52,7 @@ class Command(ScrapyCommand):
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
         parser.add_option("--cbkwargs", dest="cbkwargs",
-            help="inject extra cbkwargs into the Request, it must be a valid raw json string")
+            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",

From 1f9f41b85055c25770951c899fe839d96aa2060c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 26 Jun 2019 12:31:12 -0300
Subject: [PATCH 19/21] Move request.cb_kwargs tests to their own test file

---
 tests/spiders.py                |  61 ------------------
 tests/test_crawl.py             |  28 +--------
 tests/test_request_cb_kwargs.py | 106 ++++++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 88 deletions(-)
 create mode 100644 tests/test_request_cb_kwargs.py

diff --git a/tests/spiders.py b/tests/spiders.py
index 8c8d50ff5..7816bf7c7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -28,67 +28,6 @@ class MetaSpider(MockServerSpider):
         self.meta['close_reason'] = reason
 
 
-class KeywordArgumentsSpider(MockServerSpider):
-
-    name = 'kwargs'
-    checks = list()
-
-    def start_requests(self):
-        data = {'key': 'value', 'number': 123}
-        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
-        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
-        yield Request(self.mockserver.url('/general_without'), self.parse_general)
-        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
-        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
-        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
-        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
-
-    def parse_first(self, response, key, number):
-        self.checks.append(key == 'value')
-        self.checks.append(number == 123)
-        self.crawler.stats.inc_value('boolean_checks', 2)
-        yield response.follow(
-            self.mockserver.url('/two'),
-            self.parse_second,
-            cb_kwargs={'new_key': 'new_value'})
-
-    def parse_second(self, response, new_key):
-        self.checks.append(new_key == 'new_value')
-        self.crawler.stats.inc_value('boolean_checks')
-
-    def parse_general(self, response, **kwargs):
-        if response.url.endswith('/general_with'):
-            self.checks.append(kwargs['key'] == 'value')
-            self.checks.append(kwargs['number'] == 123)
-            self.crawler.stats.inc_value('boolean_checks', 2)
-        elif response.url.endswith('/general_without'):
-            self.checks.append(kwargs == {})
-            self.crawler.stats.inc_value('boolean_checks')
-
-    def parse_no_kwargs(self, response):
-        self.checks.append(response.url.endswith('/no_kwargs'))
-        self.crawler.stats.inc_value('boolean_checks')
-
-    def parse_default(self, response, key, number=None, default=99):
-        self.checks.append(response.url.endswith('/default'))
-        self.checks.append(key == 'value')
-        self.checks.append(number == 123)
-        self.checks.append(default == 99)
-        self.crawler.stats.inc_value('boolean_checks', 4)
-
-    def parse_takes_less(self, response, key):
-        """
-        Should raise
-        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
-        """
-
-    def parse_takes_more(self, response, key, number, other):
-        """
-        Should raise
-        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
-        """
-
-
 class FollowAllSpider(MetaSpider):
 
     name = 'follow'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 2b3e56ee9..3fc13eeb7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -4,20 +4,17 @@ import logging
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
 from scrapy.utils.python import to_unicode
-from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, KeywordArgumentsSpider, \
+from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
     BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from tests.mockserver import MockServer
 
 
 class CrawlTestCase(TestCase):
 
-    maxDiff = None
-
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
@@ -26,29 +23,6 @@ class CrawlTestCase(TestCase):
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
-    @defer.inlineCallbacks
-    def test_callback_kwargs(self):
-        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
-        self.assertTrue(all(crawler.spider.checks))
-        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
-        # check exceptions for argument mismatch
-        exceptions = {}
-        for line in log.records:
-            for key in ('takes_less', 'takes_more'):
-                if key in line.getMessage():
-                    exceptions[key] = line
-        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
-        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
-        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
-
     @defer.inlineCallbacks
     def test_follow_all(self):
         crawler = self.runner.create_crawler(FollowAllSpider)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
new file mode 100644
index 000000000..ee7117c8a
--- /dev/null
+++ b/tests/test_request_cb_kwargs.py
@@ -0,0 +1,106 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+import six
+
+from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
+from tests.spiders import MockServerSpider
+from tests.mockserver import MockServer
+
+
+class KeywordArgumentsSpider(MockServerSpider):
+
+    name = 'kwargs'
+    checks = list()
+
+    def start_requests(self):
+        data = {'key': 'value', 'number': 123}
+        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_without'), self.parse_general)
+        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
+
+    def parse_first(self, response, key, number):
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.crawler.stats.inc_value('boolean_checks', 2)
+        yield response.follow(
+            self.mockserver.url('/two'),
+            self.parse_second,
+            cb_kwargs={'new_key': 'new_value'})
+
+    def parse_second(self, response, new_key):
+        self.checks.append(new_key == 'new_value')
+        self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_general(self, response, **kwargs):
+        if response.url.endswith('/general_with'):
+            self.checks.append(kwargs['key'] == 'value')
+            self.checks.append(kwargs['number'] == 123)
+            self.crawler.stats.inc_value('boolean_checks', 2)
+        elif response.url.endswith('/general_without'):
+            self.checks.append(kwargs == {})
+            self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_no_kwargs(self, response):
+        self.checks.append(response.url.endswith('/no_kwargs'))
+        self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_default(self, response, key, number=None, default=99):
+        self.checks.append(response.url.endswith('/default'))
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.checks.append(default == 99)
+        self.crawler.stats.inc_value('boolean_checks', 4)
+
+    def parse_takes_less(self, response, key):
+        """
+        Should raise
+        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
+        """
+
+    def parse_takes_more(self, response, key, number, other):
+        """
+        Should raise
+        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
+        """
+
+
+class CallbackKeywordArgumentsTestCase(TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_callback_kwargs(self):
+        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertTrue(all(crawler.spider.checks))
+        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+        # check exceptions for argument mismatch
+        exceptions = {}
+        for line in log.records:
+            for key in ('takes_less', 'takes_more'):
+                if key in line.getMessage():
+                    exceptions[key] = line
+        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+        # py2 and py3 messages are different
+        exc_message = str(exceptions['takes_more'].exc_info[1])
+        if six.PY2:
+            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
+        elif six.PY3:
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
\ No newline at end of file

From d4d68cf32187a129b556f371f8bcc24e0ee951ba Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 26 Jun 2019 12:31:41 -0300
Subject: [PATCH 20/21] Request.cb_kwargs: update in downloader middleware

---
 tests/test_request_cb_kwargs.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index ee7117c8a..25c0bcf54 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -9,9 +9,26 @@ from tests.spiders import MockServerSpider
 from tests.mockserver import MockServer
 
 
-class KeywordArgumentsSpider(MockServerSpider):
+class InjectArgumentsDownloaderMiddleware(object):
+    def process_request(self, request, spider):
+        if request.callback.__name__ == 'parse_downloader_mw':
+            request.cb_kwargs['from_process_request'] = True
+        return None
 
+    def process_response(self, request, response, spider):
+        if request.callback.__name__ == 'parse_downloader_mw':
+            request.cb_kwargs['from_process_response'] = True
+        return response
+
+
+class KeywordArgumentsSpider(MockServerSpider):
     name = 'kwargs'
+    custom_settings = {
+        'DOWNLOADER_MIDDLEWARES': {
+            __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
+        }
+    }
+
     checks = list()
 
     def start_requests(self):
@@ -23,6 +40,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
         yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
         yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
+        yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
 
     def parse_first(self, response, key, number):
         self.checks.append(key == 'value')
@@ -69,6 +87,14 @@ class KeywordArgumentsSpider(MockServerSpider):
         TypeError: parse_takes_more() missing 1 required positional argument: 'other'
         """
 
+    def parse_downloader_mw(self, response, from_process_request, from_process_response):
+        """
+        Check if downloader middlewares are able to update the keyword arguments
+        """
+        self.checks.append(bool(from_process_request))
+        self.checks.append(bool(from_process_response))
+        self.crawler.stats.inc_value('boolean_checks', 2)
+
 
 class CallbackKeywordArgumentsTestCase(TestCase):
 

From 312e57382c2c49d9676f861ac6cf4202f69a84e5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 26 Jun 2019 12:48:00 -0300
Subject: [PATCH 21/21] Request.cb_kwargs: update in spider middleware

---
 tests/test_request_cb_kwargs.py | 47 +++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 5 deletions(-)

diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 25c0bcf54..c9943faa8 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -10,6 +10,9 @@ from tests.mockserver import MockServer
 
 
 class InjectArgumentsDownloaderMiddleware(object):
+    """
+    Make sure downloader middlewares are able to update the keyword arguments
+    """
     def process_request(self, request, spider):
         if request.callback.__name__ == 'parse_downloader_mw':
             request.cb_kwargs['from_process_request'] = True
@@ -21,12 +24,38 @@ class InjectArgumentsDownloaderMiddleware(object):
         return response
 
 
+class InjectArgumentsSpiderMiddleware(object):
+    """
+    Make sure spider middlewares are able to update the keyword arguments
+    """
+    def process_start_requests(self, start_requests, spider):
+        for request in start_requests:
+            if request.callback.__name__ == 'parse_spider_mw':
+                request.cb_kwargs['from_process_start_requests'] = True
+            yield request
+
+    def process_spider_input(self, response, spider):
+        request = response.request
+        if request.callback.__name__ == 'parse_spider_mw':
+            request.cb_kwargs['from_process_spider_input'] = True
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        for element in result:
+            if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':
+                element.cb_kwargs['from_process_spider_output'] = True
+            yield element
+
+
 class KeywordArgumentsSpider(MockServerSpider):
     name = 'kwargs'
     custom_settings = {
         'DOWNLOADER_MIDDLEWARES': {
             __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
-        }
+        },
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.InjectArgumentsSpiderMiddleware': 750,
+        },
     }
 
     checks = list()
@@ -41,6 +70,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
         yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
         yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
+        yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)
 
     def parse_first(self, response, key, number):
         self.checks.append(key == 'value')
@@ -88,13 +118,20 @@ class KeywordArgumentsSpider(MockServerSpider):
         """
 
     def parse_downloader_mw(self, response, from_process_request, from_process_response):
-        """
-        Check if downloader middlewares are able to update the keyword arguments
-        """
         self.checks.append(bool(from_process_request))
         self.checks.append(bool(from_process_response))
         self.crawler.stats.inc_value('boolean_checks', 2)
 
+    def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):
+        self.checks.append(bool(from_process_spider_input))
+        self.checks.append(bool(from_process_start_requests))
+        self.crawler.stats.inc_value('boolean_checks', 2)
+        return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)
+
+    def parse_spider_mw_2(self, response, from_process_spider_output):
+        self.checks.append(bool(from_process_spider_output))
+        self.crawler.stats.inc_value('boolean_checks', 1)
+
 
 class CallbackKeywordArgumentsTestCase(TestCase):
 
@@ -129,4 +166,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         if six.PY2:
             self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
         elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
\ No newline at end of file
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")