From 5982e3477c732f4dff5accea6eab486e4ab52c3e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Dec 2019 14:12:21 -0300
Subject: [PATCH] Take keyword arguments in base parsing methods

---
 docs/topics/spiders.rst    | 12 +++++++-----
 scrapy/spiders/__init__.py |  6 +++---
 scrapy/spiders/crawl.py    | 11 ++++++++---
 scrapy/spiders/feed.py     |  4 ++--
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b0fb14e24..dd763b607 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -362,12 +362,14 @@ CrawlSpider
 
    This spider also exposes an overrideable method:
 
-   .. method:: parse_start_url(response)
+   .. method:: parse_start_url(response, **kwargs)
 
-      This method is called for the start_urls responses. It allows to parse
-      the initial responses and must return either an
-      :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
-      object, or an iterable containing any of them.
+      This method is called for each response produced for the URLs in
+      the spider's ``start_urls`` attribute. It allows to parse
+      the initial responses and must return either an item
+      (:class:`scrapy.item.Item` or :class:`dict`),
+      a :class:`~scrapy.http.Request`,
+      or an iterable containing any of them.
 
 Crawling rules
 ~~~~~~~~~~~~~~
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 1011eb870..3e19f1e23 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -80,10 +80,10 @@ class Spider(object_ref):
         """ This method is deprecated. """
         return Request(url, dont_filter=True)
 
-    def _parse(self, response):
-        return self.parse(response)
+    def _parse(self, response, **kwargs):
+        return self.parse(response, **kwargs)
 
-    def parse(self, response):
+    def parse(self, response, **kwargs):
         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
     @classmethod
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e28d17dcd..4ec0de78c 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -74,10 +74,15 @@ class CrawlSpider(Spider):
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
-    def _parse(self, response):
-        return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
+    def _parse(self, response, **kwargs):
+        return self._parse_response(
+            response=response,
+            callback=self.parse_start_url,
+            cb_kwargs=kwargs,
+            follow=True,
+        )
 
-    def parse_start_url(self, response):
+    def parse_start_url(self, response, **kwargs):
         return []
 
     def process_results(self, response, results):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 11bd17db4..4fa6009a5 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def _parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_node'):
             raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
 
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def _parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_row'):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)