diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7195017ff..1da1a4059 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination:: next_page = response.css('li.next a::attr("href")').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + yield response.follow(next_page, self.parse) Put this in a text file, name it to something like ``quotes_spider.py`` diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3dc5ad2ed..d47bf69e5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. +As a shortcut for creating Request objects you can use +:meth:`response.follow ` method:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + ] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +Unlike scrapy.Request, ``response.follow`` supports +relative URLs directly; you can also pass a selector to it instead of +a string. Note that ``response.follow`` just returns a Request instance; +you still have to yield this Request. + More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: - import scrapy @@ -568,15 +595,12 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author + a::attr(href)').extract(): - yield scrapy.Request(response.urljoin(href), - callback=self.parse_author) + for href in response.css('.author + a::attr(href)'): + yield response.follow(href, self.parse_author) # follow pagination links - next_page = response.css('li.next a::attr(href)').extract_first() - if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, self.parse) def parse_author(self, response): def extract_with_css(query): @@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also the pagination links with the ``parse`` callback as we saw before. +Here we're passing callbacks to ``response.follow`` as positional arguments +to make the code shorter; it also works for ``scrapy.Request``. + The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument:: next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, self.parse) + yield response.follow(next_page, self.parse) If you pass the ``tag=humor`` argument to this spider, you'll notice that it diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1fdd26043..71050fddd 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -683,6 +683,10 @@ TextResponse objects response.css('p') + .. method:: TextResponse.follow(url, ...) + + Return a scrapy.Request instance to follow a link ``url``. + .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5a6507aa8..1718b1f3b 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst import six from six.moves.urllib.parse import urljoin +import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding + +from scrapy.link import Link +from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url from scrapy.utils.python import memoizemethod_noargs, to_native_str @@ -116,3 +120,58 @@ class TextResponse(Response): def css(self, query): return self.selector.css(query) + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding=None, priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object (e.g. a link extractor result); + * attribute Selector (not SelectorList) - e.g. + ``response.css('a::attr(href)')[0]`` or + ``response.xpath('//img/@src')[0]``. + * a Selector for ```` element, e.g. + ``response.css('a.my_link')[0]``. + """ + if isinstance(url, Link): + url = url.url + elif isinstance(url, parsel.Selector): + url = _url_from_selector(url) + elif isinstance(url, parsel.SelectorList): + raise ValueError("Please pass either string") + + + encoding = self.encoding if encoding is None else encoding + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) + + +def _url_from_selector(sel): + # type: (parsel.Selector) -> str + if isinstance(sel.root, six.string_types): + # e.g. ::attr(href) result + return sel.root + if not hasattr(sel.root, 'tag'): + raise ValueError("Unsupported selector: %s" % sel) + if sel.root.tag != 'a': + raise ValueError("Only elements are supported; got <%s>" % + sel.root.tag) + href = sel.root.get('href') + if href is None: + raise ValueError(" element has no href attribute: %s" % sel) + return href