mirror of https://github.com/scrapy/scrapy.git
Make tutorial doctests pass
This commit is contained in:
Adrián Chaves
parent
d267e183b4
commit
79d2f99995
|
|
@ -0,0 +1,281 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>Quotes to Scrape</title>
|
||||||
|
<link rel="stylesheet" href="/static/bootstrap.min.css">
|
||||||
|
<link rel="stylesheet" href="/static/main.css">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<div class="row header-box">
|
||||||
|
<div class="col-md-8">
|
||||||
|
<h1>
|
||||||
|
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
|
||||||
|
</h1>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<p>
|
||||||
|
|
||||||
|
<a href="/login">Login</a>
|
||||||
|
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-md-8">
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||||
|
<a href="/author/Albert-Einstein">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/change/page/1/">change</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/world/page/1/">world</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
|
||||||
|
<a href="/author/J-K-Rowling">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/choices/page/1/">choices</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||||
|
<a href="/author/Albert-Einstein">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/live/page/1/">live</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Jane Austen</small>
|
||||||
|
<a href="/author/Jane-Austen">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/books/page/1/">books</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/classic/page/1/">classic</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
|
||||||
|
<a href="/author/Marilyn-Monroe">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||||
|
<a href="/author/Albert-Einstein">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/success/page/1/">success</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/value/page/1/">value</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">André Gide</small>
|
||||||
|
<a href="/author/Andre-Gide">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="life,love" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/love/page/1/">love</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“I have not failed. I've just found 10,000 ways that won't work.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
|
||||||
|
<a href="/author/Thomas-A-Edison">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/edison/page/1/">edison</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/failure/page/1/">failure</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
|
||||||
|
<a href="/author/Eleanor-Roosevelt">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||||
|
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
|
||||||
|
<span>by <small class="author" itemprop="author">Steve Martin</small>
|
||||||
|
<a href="/author/Steve-Martin">(about)</a>
|
||||||
|
</span>
|
||||||
|
<div class="tags">
|
||||||
|
Tags:
|
||||||
|
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
|
||||||
|
|
||||||
|
<a class="tag" href="/tag/simile/page/1/">simile</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<nav>
|
||||||
|
<ul class="pager">
|
||||||
|
|
||||||
|
|
||||||
|
<li class="next">
|
||||||
|
<a href="/page/2/">Next <span aria-hidden="true">→</span></a>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4 tags-box">
|
||||||
|
|
||||||
|
<h2>Top Ten tags</h2>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span class="tag-item">
|
||||||
|
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<footer class="footer">
|
||||||
|
<div class="container">
|
||||||
|
<p class="text-muted">
|
||||||
|
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
|
||||||
|
</p>
|
||||||
|
<p class="copyright">
|
||||||
|
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</footer>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -1,16 +1,29 @@
|
||||||
from doctest import ELLIPSIS
|
import os
|
||||||
|
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
||||||
|
|
||||||
|
from scrapy.http.response.html import HtmlResponse
|
||||||
from sybil import Sybil
|
from sybil import Sybil
|
||||||
from sybil.parsers.codeblock import CodeBlockParser
|
from sybil.parsers.codeblock import CodeBlockParser
|
||||||
from sybil.parsers.doctest import DocTestParser
|
from sybil.parsers.doctest import DocTestParser
|
||||||
from sybil.parsers.skip import skip
|
from sybil.parsers.skip import skip
|
||||||
|
|
||||||
|
|
||||||
|
def load_response(url, filename):
|
||||||
|
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
|
||||||
|
with open(input_path, 'rb') as input_file:
|
||||||
|
return HtmlResponse(url, body=input_file.read())
|
||||||
|
|
||||||
|
|
||||||
|
def setup(namespace):
|
||||||
|
namespace['load_response'] = load_response
|
||||||
|
|
||||||
|
|
||||||
pytest_collect_file = Sybil(
|
pytest_collect_file = Sybil(
|
||||||
parsers=[
|
parsers=[
|
||||||
DocTestParser(optionflags=ELLIPSIS),
|
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||||
CodeBlockParser(future_imports=['print_function']),
|
CodeBlockParser(future_imports=['print_function']),
|
||||||
skip,
|
skip,
|
||||||
],
|
],
|
||||||
pattern='*.rst',
|
pattern='*.rst',
|
||||||
|
setup=setup,
|
||||||
).pytest()
|
).pytest()
|
||||||
|
|
|
||||||
|
|
@ -235,13 +235,16 @@ You will see something like::
|
||||||
[s] shelp() Shell help (print this help)
|
[s] shelp() Shell help (print this help)
|
||||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||||
[s] view(response) View response in a browser
|
[s] view(response) View response in a browser
|
||||||
>>>
|
|
||||||
|
|
||||||
Using the shell, you can try selecting elements using `CSS`_ with the response
|
Using the shell, you can try selecting elements using `CSS`_ with the response
|
||||||
object::
|
object:
|
||||||
|
|
||||||
>>> response.css('title')
|
.. invisible-code-block: python
|
||||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
|
||||||
|
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||||
|
|
||||||
|
>>> response.css('title')
|
||||||
|
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||||
|
|
||||||
The result of running ``response.css('title')`` is a list-like object called
|
The result of running ``response.css('title')`` is a list-like object called
|
||||||
:class:`~scrapy.selector.SelectorList`, which represents a list of
|
:class:`~scrapy.selector.SelectorList`, which represents a list of
|
||||||
|
|
@ -372,6 +375,9 @@ we want::
|
||||||
We get a list of selectors for the quote HTML elements with::
|
We get a list of selectors for the quote HTML elements with::
|
||||||
|
|
||||||
>>> response.css("div.quote")
|
>>> response.css("div.quote")
|
||||||
|
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||||
|
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||||
|
...]
|
||||||
|
|
||||||
Each of the selectors returned by the query above allows us to run further
|
Each of the selectors returned by the query above allows us to run further
|
||||||
queries over their sub-elements. Let's assign the first selector to a
|
queries over their sub-elements. Let's assign the first selector to a
|
||||||
|
|
@ -404,10 +410,9 @@ quotes elements and put them together into a Python dictionary::
|
||||||
... author = quote.css("small.author::text").get()
|
... author = quote.css("small.author::text").get()
|
||||||
... tags = quote.css("div.tags a.tag::text").getall()
|
... tags = quote.css("div.tags a.tag::text").getall()
|
||||||
... print(dict(text=text, author=author, tags=tags))
|
... print(dict(text=text, author=author, tags=tags))
|
||||||
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
|
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
|
||||||
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
|
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
|
||||||
... a few more of these, omitted for brevity
|
...
|
||||||
>>>
|
|
||||||
|
|
||||||
Extracting data in our spider
|
Extracting data in our spider
|
||||||
-----------------------------
|
-----------------------------
|
||||||
|
|
@ -521,7 +526,7 @@ There is also an ``attrib`` property available
|
||||||
(see :ref:`selecting-attributes` for more)::
|
(see :ref:`selecting-attributes` for more)::
|
||||||
|
|
||||||
>>> response.css('li.next a').attrib['href']
|
>>> response.css('li.next a').attrib['href']
|
||||||
'/page/2'
|
'/page/2/'
|
||||||
|
|
||||||
Let's see now our spider modified to recursively follow the link to the next
|
Let's see now our spider modified to recursively follow the link to the next
|
||||||
page, extracting data from it::
|
page, extracting data from it::
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ addopts =
|
||||||
--doctest-modules
|
--doctest-modules
|
||||||
--ignore=docs/_ext
|
--ignore=docs/_ext
|
||||||
--ignore=docs/conf.py
|
--ignore=docs/conf.py
|
||||||
--ignore=docs/intro/tutorial.rst
|
|
||||||
--ignore=docs/news.rst
|
--ignore=docs/news.rst
|
||||||
--ignore=docs/topics/commands.rst
|
--ignore=docs/topics/commands.rst
|
||||||
--ignore=docs/topics/debug.rst
|
--ignore=docs/topics/debug.rst
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue