initial response.follow implementation

docs/intro/overview.rst

@@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination::
 
             next_page = response.css('li.next a::attr("href")').extract_first()
             if next_page is not None:
-                next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, callback=self.parse)
+                yield response.follow(next_page, self.parse)
 
 
 Put this in a text file, name it to something like ``quotes_spider.py``

docs/intro/tutorial.rst

@@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p
 until it doesn't find one -- handy for crawling blogs, forums and other sites with
 pagination.
 
+As a shortcut for creating Request objects you can use
+:meth:`response.follow <scrapy.http.TextResponse.follow>` method::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = "quotes"
+        start_urls = [
+            'http://quotes.toscrape.com/page/1/',
+        ]
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {
+                    'text': quote.css('span.text::text').extract_first(),
+                    'author': quote.css('span small::text').extract_first(),
+                    'tags': quote.css('div.tags a.tag::text').extract(),
+                }
+
+            for href in response.css('li.next a::attr(href)'):
+                yield response.follow(href, callback=self.parse)
+
+Unlike scrapy.Request, ``response.follow`` supports
+relative URLs directly; you can also pass a selector to it instead of
+a string. Note that ``response.follow`` just returns a Request instance;
+you still have to yield this Request.
+
 More examples and patterns
 --------------------------
 
 Here is another spider that illustrates callbacks and following links,
 this time for scraping author information::
 
-
     import scrapy
 
 
@@ -568,15 +595,12 @@ this time for scraping author information::
 
         def parse(self, response):
             # follow links to author pages
-            for href in response.css('.author + a::attr(href)').extract():
-                yield scrapy.Request(response.urljoin(href),
-                                     callback=self.parse_author)
+            for href in response.css('.author + a::attr(href)'):
+                yield response.follow(href, self.parse_author)
 
             # follow pagination links
-            next_page = response.css('li.next a::attr(href)').extract_first()
-            if next_page is not None:
-                next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, callback=self.parse)
+            for href in response.css('li.next a::attr(href)'):
+                yield response.follow(href, self.parse)
 
         def parse_author(self, response):
             def extract_with_css(query):
@@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the
 authors pages calling the ``parse_author`` callback for each of them, and also
 the pagination links with the ``parse`` callback as we saw before.
 
+Here we're passing callbacks to ``response.follow`` as positional arguments
+to make the code shorter; it also works for ``scrapy.Request``.
+
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.
 
@@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument::
 
             next_page = response.css('li.next a::attr(href)').extract_first()
             if next_page is not None:
-                next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, self.parse)
+                yield response.follow(next_page, self.parse)
 
 
 If you pass the ``tag=humor`` argument to this spider, you'll notice that it

docs/topics/request-response.rst

@@ -683,6 +683,10 @@ TextResponse objects
 
             response.css('p')
 
+    .. method:: TextResponse.follow(url, ...)
+
+        Return a scrapy.Request instance to follow a link ``url``.
+
     .. method:: TextResponse.body_as_unicode()
 
         The same as :attr:`text`, but available as a method. This method is

scrapy/http/response/text.py

@@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst
 import six
 from six.moves.urllib.parse import urljoin
 
+import parsel
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
+
+from scrapy.link import Link
+from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
 from scrapy.utils.python import memoizemethod_noargs, to_native_str
@@ -116,3 +120,58 @@ class TextResponse(Response):
 
     def css(self, query):
         return self.selector.css(query)
+
+    def follow(self, url, callback=None, method='GET', headers=None, body=None,
+               cookies=None, meta=None, encoding=None, priority=0,
+               dont_filter=False, errback=None):
+        # type: (...) -> Request
+        """
+        Return a scrapy.Request instance to follow a link ``url``.
+
+        ``url`` can be:
+
+        * absolute URL;
+        * relative URL;
+        * scrapy.link.Link object (e.g. a link extractor result);
+        * attribute Selector (not SelectorList) - e.g.
+          ``response.css('a::attr(href)')[0]`` or
+          ``response.xpath('//img/@src')[0]``.
+        * a Selector for ``<a>`` element, e.g.
+          ``response.css('a.my_link')[0]``.
+        """
+        if isinstance(url, Link):
+            url = url.url
+        elif isinstance(url, parsel.Selector):
+            url = _url_from_selector(url)
+        elif isinstance(url, parsel.SelectorList):
+            raise ValueError("Please pass either string")
+
+
+        encoding = self.encoding if encoding is None else encoding
+        url = self.urljoin(url)
+        return Request(url, callback,
+                       method=method,
+                       headers=headers,
+                       body=body,
+                       cookies=cookies,
+                       meta=meta,
+                       encoding=encoding,
+                       priority=priority,
+                       dont_filter=dont_filter,
+                       errback=errback)
+
+
+def _url_from_selector(sel):
+    # type: (parsel.Selector) -> str
+    if isinstance(sel.root, six.string_types):
+        # e.g. ::attr(href) result
+        return sel.root
+    if not hasattr(sel.root, 'tag'):
+        raise ValueError("Unsupported selector: %s" % sel)
+    if sel.root.tag != 'a':
+        raise ValueError("Only <a> elements are supported; got <%s>" %
+                         sel.root.tag)
+    href = sel.root.get('href')
+    if href is None:
+        raise ValueError("<a> element has no href attribute: %s" % sel)
+    return href