From 4626e90df8ba4a945bb9cd6be47a915788e76f23 Mon Sep 17 00:00:00 2001
From: Abhishek Pratap Singh <35230163+Prime-5@users.noreply.github.com>
Date: Mon, 10 Feb 2020 18:48:31 +0000
Subject: [PATCH] Allow updating flags in follow and follow_all (#4279)

---
 scrapy/http/response/__init__.py |  7 +++++--
 scrapy/http/response/text.py     |  6 ++++--
 tests/test_http_response.py      | 31 +++++++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index f92d0901c..027fbac6f 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -107,7 +107,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -124,6 +124,7 @@ class Response(object_ref):
         elif url is None:
             raise ValueError("url can't be None")
         url = self.urljoin(url)
+
         return Request(
             url=url,
             callback=callback,
@@ -137,11 +138,12 @@ class Response(object_ref):
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
     def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding='utf-8', priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None):
+                   dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Generator[Request, None, None]
         """
         Return an iterable of :class:`~.Request` instances to follow all links
@@ -169,6 +171,7 @@ class Response(object_ref):
                 dont_filter=dont_filter,
                 errback=errback,
                 cb_kwargs=cb_kwargs,
+                flags=flags,
             )
             for url in urls
         )
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 09049c157..33a485328 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -121,7 +121,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -157,11 +157,12 @@ class TextResponse(Response):
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
     def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding=None, priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None,
+                   dont_filter=False, errback=None, cb_kwargs=None, flags=None,
                    css=None, xpath=None):
         # type: (...) -> Generator[Request, None, None]
         """
@@ -214,6 +215,7 @@ class TextResponse(Response):
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 4c1b2afc3..ff487cfa3 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -166,6 +166,10 @@ class BaseResponseTest(unittest.TestCase):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
+    def test_follow_flags(self):
+        res = self.response_class('http://example.com/')
+        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
+        self.assertEqual(fol.flags, ['cached', 'allowed'])
 
     # Response.follow_all
 
@@ -232,6 +236,17 @@ class BaseResponseTest(unittest.TestCase):
         expected = [u.replace(' ', '%20') for u in absolute]
         self._assert_followed_all_urls(links, expected)
 
+    def test_follow_all_flags(self):
+        re = self.response_class('http://www.example.com/')
+        urls = [
+            'http://www.example.com/',
+            'http://www.example.com/2',
+            'http://www.example.com/foo',
+        ]
+        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        for req in fol:
+            self.assertEqual(req.flags, ['cached', 'allowed'])
+
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
             response = self._links_response()
@@ -562,6 +577,22 @@ class TextResponseTest(BaseResponseTest):
         )
         self.assertEqual(req.encoding, 'cp1251')
 
+    def test_follow_flags(self):
+        res = self.response_class('http://example.com/')
+        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
+        self.assertEqual(fol.flags, ['cached', 'allowed'])
+
+    def test_follow_all_flags(self):
+        re = self.response_class('http://www.example.com/')
+        urls = [
+            'http://www.example.com/',
+            'http://www.example.com/2',
+            'http://www.example.com/foo',
+        ]
+        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        for req in fol:
+            self.assertEqual(req.flags, ['cached', 'allowed'])
+
     def test_follow_all_css(self):
         expected = [
             'http://example.com/sample3.html',