RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into from_crawler_downloader_handlers

This commit is contained in:
Eugenio Lacuesta 2019-12-21 17:24:45 -03:00
parent e43f37fff3 5577d4d2be
commit 310cc081d0
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
187 changed files with 2034 additions and 1498 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.bandit.yml
View File

@ -1,13 +1,15 @@
skips:
- B101
- B105
- B301
- B303
- B306
- B307
- B311
- B320
- B321
- B402
- B402 # https://github.com/scrapy/scrapy/issues/4180
- B403
- B404
- B406
- B410

9
.readthedocs.yml Normal file
View File

@ -0,0 +1,9 @@
version: 2
sphinx:
configuration: docs/conf.py
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.7 # Keep in sync with .travis.yml
install:
- requirements: docs/requirements.txt

7
.travis.yml
View File

@ -12,10 +12,9 @@ matrix:
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
python: 3.5
- env: TOXENV=py35
python: 3.5
- env: TOXENV=py35-pinned
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py36
python: 3.6
@ -23,10 +22,10 @@ matrix:
python: 3.7
- env: TOXENV=py38
python: 3.8
- env: TOXENV=py38-extra-deps
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=docs
python: 3.6
python: 3.7 # Keep in sync with .readthedocs.yml
install:
- |
if [ "$TOXENV" = "pypy3" ]; then

9
conftest.py
View File

@ -1,12 +1,19 @@
from pathlib import Path
import pytest
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess")
]
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':

281
docs/_tests/quotes.html Normal file
View File

@ -0,0 +1,281 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Quotes to Scrape</title>
<link rel="stylesheet" href="/static/bootstrap.min.css">
<link rel="stylesheet" href="/static/main.css">
</head>
<body>
<div class="container">
<div class="row header-box">
<div class="col-md-8">
<h1>
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
</h1>
</div>
<div class="col-md-4">
<p>
<a href="/login">Login</a>
</p>
</div>
</div>
<div class="row">
<div class="col-md-8">
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
<a class="tag" href="/tag/change/page/1/">change</a>
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
<a class="tag" href="/tag/world/page/1/">world</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
<a href="/author/J-K-Rowling">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
<a class="tag" href="/tag/choices/page/1/">choices</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/live/page/1/">live</a>
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
<span>by <small class="author" itemprop="author">Jane Austen</small>
<a href="/author/Jane-Austen">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
<a class="tag" href="/tag/books/page/1/">books</a>
<a class="tag" href="/tag/classic/page/1/">classic</a>
<a class="tag" href="/tag/humor/page/1/">humor</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
<a href="/author/Marilyn-Monroe">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
<a class="tag" href="/tag/success/page/1/">success</a>
<a class="tag" href="/tag/value/page/1/">value</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
<span>by <small class="author" itemprop="author">André Gide</small>
<a href="/author/Andre-Gide">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="life,love" / >
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/love/page/1/">love</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
<a href="/author/Thomas-A-Edison">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
<a class="tag" href="/tag/edison/page/1/">edison</a>
<a class="tag" href="/tag/failure/page/1/">failure</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
<a href="/author/Eleanor-Roosevelt">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
<span>by <small class="author" itemprop="author">Steve Martin</small>
<a href="/author/Steve-Martin">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
<a class="tag" href="/tag/humor/page/1/">humor</a>
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
<a class="tag" href="/tag/simile/page/1/">simile</a>
</div>
</div>
<nav>
<ul class="pager">
<li class="next">
<a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
</li>
</ul>
</nav>
</div>
<div class="col-md-4 tags-box">
<h2>Top Ten tags</h2>
<span class="tag-item">
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
</span>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="text-muted">
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
</p>
</div>
</footer>
</body>
</html>

13
docs/conf.py
View File

@ -12,6 +12,7 @@
# serve to show the default.
import sys
from datetime import datetime
from os import path
# If your extensions are in another directory, add it here. If the directory
@ -49,8 +50,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'Scrapy'
copyright = u'2008–2018, Scrapy developers'
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -194,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', u'Scrapy Documentation',
u'Scrapy developers', 'manual'),
('index', 'Scrapy.tex', 'Scrapy Documentation',
'Scrapy developers', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@ -268,6 +269,10 @@ coverage_ignore_pyobjects = [
# Never documented before, and deprecated now.
r'^scrapy\.item\.DictItem$',
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
# Implementation detail of LxmlLinkExtractor
r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
]

18
docs/contributing.rst
View File

@ -44,7 +44,7 @@ guidelines when you're going to report a new bug.
* check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
well-known question
* if you have a general question about scrapy usage, please ask it at
* if you have a general question about Scrapy usage, please ask it at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
(use "scrapy" tag).
@ -203,17 +203,9 @@ Tests are implemented using the :doc:`Twisted unit-testing framework
Running tests
-------------
Make sure you have a recent enough :doc:`tox <tox:index>` installation:
To run all tests::
``tox --version``
If your version is older than 1.7.0, please update it first:
``pip install -U tox``
To run all tests go to the root directory of Scrapy source code and run:
``tox``
tox
To run a specific test (say ``tests/test_loader.py``) use:
@ -225,11 +217,11 @@ the tests with Python 3.6 use::
tox -e py36
You can also specify a comma-separated list of environmets, and use :ref:`tox’s
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py27,py36 -p auto
tox -e py36,py38 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the

2
docs/faq.rst
View File

@ -338,7 +338,7 @@ How to split an item into multiple items in an item pipeline?
input item. :ref:`Create a spider middleware <custom-spider-middleware>`
instead, and use its
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method for this puspose. For example::
method for this purpose. For example::
from copy import deepcopy

2
docs/index.rst
View File

@ -170,7 +170,7 @@ Solving specific problems
Get answers to most frequently asked questions.
:doc:`topics/debug`
Learn how to debug common problems of your scrapy spider.
Learn how to debug common problems of your Scrapy spider.
:doc:`topics/contracts`
Learn how to use contracts for testing your spiders.

16
docs/intro/install.rst
View File

@ -78,9 +78,9 @@ TL;DR: We recommend installing Scrapy inside a virtual environment
on all platforms.
Python packages can be installed either globally (a.k.a system wide),
or in user-space. We do not recommend installing scrapy system wide.
or in user-space. We do not recommend installing Scrapy system wide.
Instead, we recommend that you install scrapy within a so-called
Instead, we recommend that you install Scrapy within a so-called
"virtual environment" (`virtualenv`_).
Virtualenvs allow you to not conflict with already-installed Python
system packages (which could break some of your system tools and scripts),
@ -97,7 +97,7 @@ Check this `user guide`_ on how to create your virtualenv.
.. note::
If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
Once you have created a virtualenv, you can install scrapy inside it with ``pip``,
Once you have created a virtualenv, you can install Scrapy inside it with ``pip``,
just like any other Python package.
(See :ref:`platform-specific guides <intro-install-platform-notes>`
below for non-Python dependencies that you may need to install beforehand).
@ -144,7 +144,7 @@ albeit with potential issues with TLS connections.
typically too old and slow to catch up with latest Scrapy.
To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
these dependencies::
sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
@ -225,17 +225,17 @@ PyPy
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
On OS X, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
recommends (only needed when installing scrapy). Installing on Linux has no special
recommends (only needed when installing Scrapy). Installing on Linux has no special
issues besides installing build dependencies.
Installing scrapy with PyPy on Windows is not tested.
Installing Scrapy with PyPy on Windows is not tested.
You can check that scrapy is installed correctly by running ``scrapy bench``.
You can check that Scrapy is installed correctly by running ``scrapy bench``.
If this command gives errors such as
``TypeError: ... got 2 unexpected keyword arguments``, this means
that setuptools was unable to pick up one PyPy-specific dependency.

2
docs/intro/overview.rst
View File

@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.xpath('span/small/text()').get(),
'text': quote.css('span.text::text').get(),
}
next_page = response.css('li.next a::attr("href")').get()

120
docs/intro/tutorial.rst
View File

@ -252,30 +252,30 @@ The result of running ``response.css('title')`` is a list-like object called
and allow you to run further queries to fine-grain the selection or extract the
data.
To extract the text from the title above, you can do::
To extract the text from the title above, you can do:
>>> response.css('title::text').getall()
['Quotes to Scrape']
>>> response.css('title::text').getall()
['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
CSS query, to mean we want to select only the text elements directly inside
``<title>`` element. If we don't specify ``::text``, we'd get the full title
element, including its tags::
element, including its tags:
>>> response.css('title').getall()
['<title>Quotes to Scrape</title>']
>>> response.css('title').getall()
['<title>Quotes to Scrape</title>']
The other thing is that the result of calling ``.getall()`` is a list: it is
possible that a selector returns more than one result, so we extract them all.
When you know you just want the first result, as in this case, you can do::
When you know you just want the first result, as in this case, you can do:
>>> response.css('title::text').get()
'Quotes to Scrape'
>>> response.css('title::text').get()
'Quotes to Scrape'
As an alternative, you could've written::
As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
@ -288,14 +288,14 @@ to be scraped, you can at least get **some** data.
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
expressions`_::
expressions`_:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
In order to find the proper CSS selectors to use, you might find useful opening
the response page from the shell in your web browser using ``view(response)``.
@ -312,12 +312,12 @@ visually selected elements, which works in many browsers.
XPath: a brief intro
^^^^^^^^^^^^^^^^^^^^
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').get()
'Quotes to Scrape'
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').get()
'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@ -372,35 +372,35 @@ we want::
$ scrapy shell 'http://quotes.toscrape.com'
We get a list of selectors for the quote HTML elements with::
We get a list of selectors for the quote HTML elements with:
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
variable, so that we can run our CSS selectors directly on a particular quote::
variable, so that we can run our CSS selectors directly on a particular quote:
>>> quote = response.css("div.quote")[0]
>>> quote = response.css("div.quote")[0]
Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created::
using the ``quote`` object we just created:
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
Given that the tags are a list of strings, we can use the ``.getall()`` method
to get all of them::
to get all of them:
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. invisible-code-block: python
@ -409,16 +409,16 @@ to get all of them::
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary::
quotes elements and put them together into a Python dictionary:
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
Extracting data in our spider
-----------------------------
@ -516,23 +516,23 @@ markup:
</li>
</ul>
We can try extracting it in the shell::
We can try extracting it in the shell:
>>> response.css('li.next a').get()
'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
>>> response.css('li.next a').get()
'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
This gets the anchor element, but we want the attribute ``href``. For that,
Scrapy supports a CSS extension that lets you select the attribute contents,
like this::
like this:
>>> response.css('li.next a::attr(href)').get()
'/page/2/'
>>> response.css('li.next a::attr(href)').get()
'/page/2/'
There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more)::
(see :ref:`selecting-attributes` for more):
>>> response.css('li.next a').attrib['href']
'/page/2/'
>>> response.css('li.next a').attrib['href']
'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::

97
docs/news.rst
View File

@ -47,13 +47,13 @@ Backward-incompatible changes
(:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
into lists::
into lists:
>>> item = MyItem()
>>> item['field'] = 'value1'
>>> loader = ItemLoader(item=item)
>>> item['field']
['value1']
>>> item = MyItem()
>>> item['field'] = 'value1'
>>> loader = ItemLoader(item=item)
>>> item['field']
['value1']
This is needed to allow adding values to existing fields
(``loader.add_value('field', 'value2')``).
@ -678,7 +678,7 @@ Usability improvements
* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
* non-zero exit code is returned from Scrapy commands when error happens
on spider inititalization (:issue:`3226`)
on spider initialization (:issue:`3226`)
* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
"flv" is added to common video extensions (:issue:`3165`)
* better error message when an exporter is disabled (:issue:`3358`);
@ -1156,7 +1156,7 @@ Bug fixes
- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`).
- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`).
- Fix name of generated pipeline class when using basic project template (:issue:`2466`).
- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`).
- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
@ -1164,7 +1164,7 @@ Bug fixes
Documentation
~~~~~~~~~~~~~
- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4
- Reword Code of Conduct section and upgrade to Contributor Covenant v1.4
(:issue:`2469`).
- Clarify that passing spider arguments converts them to spider attributes
(:issue:`2483`).
@ -1178,7 +1178,7 @@ Documentation
Cleanups
~~~~~~~~
- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
- Remove redundant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`).
- Remove dead code supporting old Twisted versions (:issue:`2544`).
@ -1204,7 +1204,7 @@ New Features
- ``MailSender`` now accepts single strings as values for ``to`` and ``cc``
arguments (:issue:`2272`)
- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside
scrapy shell now follow HTTP redirections by default (:issue:`2290`);
Scrapy shell now follow HTTP redirections by default (:issue:`2290`);
See :command:`fetch` and :command:`shell` for details.
- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
this is technically **backward incompatible** so please check your log parsers.
@ -1360,7 +1360,7 @@ Documentation
- Grammar fixes: :issue:`2128`, :issue:`1566`.
- Download stats badge removed from README (:issue:`2160`).
- New scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`).
- New Scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`).
- Updated ``Response`` parameters documentation (:issue:`2197`).
- Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`).
- Add StackOverflow as a support channel (:issue:`2257`).
@ -1450,7 +1450,7 @@ Documentation
- Use "url" variable in downloader middleware example (:issue:`2015`)
- Grammar fixes (:issue:`2054`, :issue:`2120`)
- New FAQ entry on using BeautifulSoup in spider callbacks (:issue:`2048`)
- Add notes about scrapy not working on Windows with Python 3 (:issue:`2060`)
- Add notes about Scrapy not working on Windows with Python 3 (:issue:`2060`)
- Encourage complete titles in pull requests (:issue:`2026`)
Tests
@ -1509,7 +1509,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
You can use :setting:`FILES_STORE_S3_ACL` to change it.
- We've reimplemented ``canonicalize_url()`` for more correct output,
especially for URLs with non-ASCII characters (:issue:`1947`).
This could change link extractors output compared to previous scrapy versions.
This could change link extractors output compared to previous Scrapy versions.
This may also invalidate some cache entries you could still have from pre-1.1 runs.
**Warning: backward incompatible!**.
@ -1705,7 +1705,7 @@ Scrapy 1.0.4 (2015-12-30)
- fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`)
- Typos corrections (:commit:`7067117`)
- fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`)
- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`)
- Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`)
- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`)
- Reference Homebrew's homepage for installation instructions (:commit:`1925db1`)
- Add oldest supported tox version to contributing docs (:commit:`5d10d6d`)
@ -1722,7 +1722,7 @@ Scrapy 1.0.4 (2015-12-30)
- Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`)
- Typo (:commit:`f8d0682`)
- Fix list formatting (:commit:`5f83a93`)
- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
- fix Scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
- Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`)
- Update tutorial.rst (:commit:`fbc1f25`)
- Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`)
@ -1734,7 +1734,7 @@ Scrapy 1.0.4 (2015-12-30)
Scrapy 1.0.3 (2015-08-11)
-------------------------
- add service_identity to scrapy install_requires (:commit:`cbc2501`)
- add service_identity to Scrapy install_requires (:commit:`cbc2501`)
- Workaround for travis#296 (:commit:`66af9cd`)
.. _release-1.0.2:
@ -1758,7 +1758,7 @@ Scrapy 1.0.1 (2015-07-01)
- include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`)
- DOC Fix SelectJmes documentation (:commit:`b8567bc`)
- DOC Bring Ubuntu and Archlinux outside of Windows subsection (:commit:`392233f`)
- DOC remove version suffix from ubuntu package (:commit:`5303c66`)
- DOC remove version suffix from Ubuntu package (:commit:`5303c66`)
- DOC Update release date for 1.0 (:commit:`c89fa29`)
.. _release-1.0.0:
@ -2211,7 +2211,7 @@ Scrapy 0.24.2 (2014-07-08)
- Use a mutable mapping to proxy deprecated settings.overrides and settings.defaults attribute (:commit:`e5e8133`)
- there is not support for python3 yet (:commit:`3cd6146`)
- Update python compatible version set to debian packages (:commit:`fa5d76b`)
- Update python compatible version set to Debian packages (:commit:`fa5d76b`)
- DOC fix formatting in release notes (:commit:`c6a9e20`)
Scrapy 0.24.1 (2014-06-27)
@ -2229,12 +2229,12 @@ Enhancements
- Improve Scrapy top-level namespace (:issue:`494`, :issue:`684`)
- Add selector shortcuts to responses (:issue:`554`, :issue:`690`)
- Add new lxml based LinkExtractor to replace unmantained SgmlLinkExtractor
- Add new lxml based LinkExtractor to replace unmaintained SgmlLinkExtractor
(:issue:`559`, :issue:`761`, :issue:`763`)
- Cleanup settings API - part of per-spider settings **GSoC project** (:issue:`737`)
- Add UTF8 encoding header to templates (:issue:`688`, :issue:`762`)
- Telnet console now binds to 127.0.0.1 by default (:issue:`699`)
- Update debian/ubuntu install instructions (:issue:`509`, :issue:`549`)
- Update Debian/Ubuntu install instructions (:issue:`509`, :issue:`549`)
- Disable smart strings in lxml XPath evaluations (:issue:`535`)
- Restore filesystem based cache as default for http
cache middleware (:issue:`541`, :issue:`500`, :issue:`571`)
@ -2267,7 +2267,7 @@ Enhancements
- Tests and docs for ``request_fingerprint`` function (:issue:`597`)
- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`)
- Set exit code to non-zero when contracts fails (:issue:`727`)
- Add a setting to control what class is instanciated as Downloader component
- Add a setting to control what class is instantiated as Downloader component
(:issue:`738`)
- Pass response in ``item_dropped`` signal (:issue:`724`)
- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`)
@ -2276,7 +2276,7 @@ Enhancements
- Add a note about reporting security issues (:issue:`697`)
- Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
- Sort spider list output of ``scrapy list`` command (:issue:`742`)
- Multiple documentation enhancemens and fixes
- Multiple documentation enhancements and fixes
(:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
:issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,
:issue:`654`, :issue:`675`, :issue:`663`, :issue:`711`, :issue:`714`)
@ -2321,19 +2321,19 @@ Scrapy 0.22.1 (released 2014-02-08)
- BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
- BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
- Fix tests for Travis-CI build (:commit:`76c7e20`)
- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
- Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
- Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
- PEP8 minor edits. (:commit:`f89efaf`)
- Expose current crawler in the scrapy shell. (:commit:`5349cec`)
- Expose current crawler in the Scrapy shell. (:commit:`5349cec`)
- Unused re import and PEP8 minor edits. (:commit:`387f414`)
- Ignore None's values when using the ItemLoader. (:commit:`0632546`)
- DOC Fixed HTTPCACHE_STORAGE typo in the default value which is now Filesystem instead Dbm. (:commit:`cde9a8c`)
- show ubuntu setup instructions as literal code (:commit:`fb5c9c5`)
- show Ubuntu setup instructions as literal code (:commit:`fb5c9c5`)
- Update Ubuntu installation instructions (:commit:`70fb105`)
- Merge pull request #550 from stray-leone/patch-1 (:commit:`6f70b6a`)
- modify the version of scrapy ubuntu package (:commit:`725900d`)
- modify the version of Scrapy Ubuntu package (:commit:`725900d`)
- fix 0.22.0 release date (:commit:`af0219a`)
- fix typos in news.rst and remove (not released yet) header (:commit:`b7f58f4`)
@ -2354,7 +2354,7 @@ Enhancements
- Improve test coverage and forthcoming Python 3 support (:issue:`525`)
- Promote startup info on settings and middleware to INFO level (:issue:`520`)
- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`)
- Allow running indiviual tests via tox (:issue:`503`)
- Allow running individual tests via tox (:issue:`503`)
- Update extensions ignored by link extractors (:issue:`498`)
- Add middleware methods to get files/images/thumbs paths (:issue:`490`)
- Improve offsite middleware tests (:issue:`478`)
@ -2411,7 +2411,7 @@ Enhancements
- scrapy.mail.MailSender now can connect over TLS or upgrade using STARTTLS (:issue:`327`)
- New FilesPipeline with functionality factored out from ImagesPipeline (:issue:`370`, :issue:`409`)
- Recommend Pillow instead of PIL for image handling (:issue:`317`)
- Added debian packages for Ubuntu quantal and raring (:commit:`86230c0`)
- Added Debian packages for Ubuntu Quantal and Raring (:commit:`86230c0`)
- Mock server (used for tests) can listen for HTTPS requests (:issue:`410`)
- Remove multi spider support from multiple core components
(:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
@ -2430,7 +2430,7 @@ Bugfixes
- Fix tests under Django 1.6 (:commit:`b6bed44c`)
- Lot of bugfixes to retry middleware under disconnections using HTTP 1.1 download handler
- Fix inconsistencies among Twisted releases (:issue:`406`)
- Fix scrapy shell bugs (:issue:`418`, :issue:`407`)
- Fix Scrapy shell bugs (:issue:`418`, :issue:`407`)
- Fix invalid variable name in setup.py (:issue:`429`)
- Fix tutorial references (:issue:`387`)
- Improve request-response docs (:issue:`391`)
@ -2512,15 +2512,15 @@ Scrapy 0.18.1 (released 2013-08-27)
- test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`)
- Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`)
- do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`)
- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
- New HTTP client wraps connection lost in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
- limit travis-ci build matrix (:commit:`3b01bb8`)
- Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`)
- Fixed so it refers to the correct folder (:commit:`3283809`)
- added quantal & raring to support ubuntu releases (:commit:`1411923`)
- added Quantal & Raring to support Ubuntu releases (:commit:`1411923`)
- fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`)
- fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`)
- minor updates to 0.18 release notes (:commit:`c45e5f1`)
- fix contributters list format (:commit:`0b60031`)
- fix contributors list format (:commit:`0b60031`)
Scrapy 0.18.0 (released 2013-08-09)
-----------------------------------
@ -2555,8 +2555,8 @@ Scrapy 0.18.0 (released 2013-08-09)
- Collect idle downloader slots (:issue:`297`)
- Add ``ftp://`` scheme downloader handler (:issue:`329`)
- Added downloader benchmark webserver and spider tools :ref:`benchmarking`
- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
- Add scrapy commands using external libraries (:issue:`260`)
- Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on
- Add Scrapy commands using external libraries (:issue:`260`)
- Added ``--pdb`` option to ``scrapy`` command line tool
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
- Several improvements to spider contracts
@ -2568,7 +2568,7 @@ Scrapy 0.18.0 (released 2013-08-09)
- several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez)
- support custom download slots
- added --spider option to "shell" command.
- log overridden settings when scrapy starts
- log overridden settings when Scrapy starts
Thanks to everyone who contribute to this release. Here is a list of
contributors sorted by number of commits::
@ -2617,7 +2617,7 @@ contributors sorted by number of commits::
Scrapy 0.16.5 (released 2013-05-30)
-----------------------------------
- obey request method when scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
- obey request method when Scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
- fix inaccurate downloader middleware documentation. refs #280 (:commit:`40667cb`)
- doc: remove links to diveintopython.org, which is no longer available. closes #246 (:commit:`bd58bfa`)
- Find form nodes in invalid html5 documents (:commit:`e3d6945`)
@ -2631,8 +2631,8 @@ Scrapy 0.16.4 (released 2013-01-23)
- Fixed error message formatting. log.err() doesn't support cool formatting and when error occurred, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`)
- lint and improve images pipeline error logging (:commit:`56b45fc`)
- fixed doc typos (:commit:`243be84`)
- add documentation topics: Broad Crawls & Common Practies (:commit:`1fbb715`)
- fix bug in scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
- add documentation topics: Broad Crawls & Common Practices (:commit:`1fbb715`)
- fix bug in Scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
- Update docs/topics/commands.rst (:commit:`28eac7a`)
Scrapy 0.16.3 (released 2012-12-07)
@ -2651,11 +2651,11 @@ Scrapy 0.16.3 (released 2012-12-07)
Scrapy 0.16.2 (released 2012-11-09)
-----------------------------------
- scrapy contracts: python2.6 compat (:commit:`a4a9199`)
- scrapy contracts verbose option (:commit:`ec41673`)
- proper unittest-like output for scrapy contracts (:commit:`86635e4`)
- Scrapy contracts: python2.6 compat (:commit:`a4a9199`)
- Scrapy contracts verbose option (:commit:`ec41673`)
- proper unittest-like output for Scrapy contracts (:commit:`86635e4`)
- added open_in_browser to debugging doc (:commit:`c9b690d`)
- removed reference to global scrapy stats from settings doc (:commit:`dd55067`)
- removed reference to global Scrapy stats from settings doc (:commit:`dd55067`)
- Fix SpiderState bug in Windows platforms (:commit:`58998f4`)
@ -2665,7 +2665,7 @@ Scrapy 0.16.1 (released 2012-10-26)
- fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`)
- better backward compatibility for scrapy.conf.settings (:commit:`3403089`)
- extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`)
- removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`)
- removed .hgtags (no longer needed now that Scrapy uses git) (:commit:`d52c188`)
- fix dashes under rst headers (:commit:`fa4f7f9`)
- set release date for 0.16.0 in news (:commit:`e292246`)
@ -2680,8 +2680,7 @@ Scrapy changes:
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info.
- dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
- documented :ref:`topics-api`
- ``lxml`` is now the default selectors backend instead of ``libxml2``
@ -2715,7 +2714,7 @@ Scrapy changes:
Scrapy 0.14.4
-------------
- added precise to supported ubuntu distros (:commit:`b7e46df`)
- added precise to supported Ubuntu distros (:commit:`b7e46df`)
- fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`)
- meta tag attributes for content-type http equiv can be in any order. #123 (:commit:`0cb68af`)
- replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`)
@ -2728,11 +2727,11 @@ Scrapy 0.14.3
- include egg files used by testsuite in source distribution. #118 (:commit:`c897793`)
- update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`)
- added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`)
- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
- don't discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
- do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`)
- fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`)
- fixed minor defect in link extractors documentation (:commit:`ba14f38`)
- removed some obsolete remaining code related to sqlite support in scrapy (:commit:`0665175`)
- removed some obsolete remaining code related to sqlite support in Scrapy (:commit:`0665175`)
Scrapy 0.14.2
-------------

1
docs/requirements.txt
View File

@ -1,4 +1,3 @@
-r ../requirements-py3.txt
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page

2
docs/topics/autothrottle.rst
View File

@ -11,7 +11,7 @@ Design goals
============
1. be nicer to sites instead of using default download delay of zero
2. automatically adjust scrapy to the optimum crawling speed, so the user
2. automatically adjust Scrapy to the optimum crawling speed, so the user
doesn't have to tune the download delays to find the optimum one.
The user only needs to specify the maximum concurrent requests
it allows, and the extension does the rest.

2
docs/topics/commands.rst
View File

@ -29,7 +29,7 @@ in standard locations:
1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
for global (user-wide) settings, and
3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
Settings from these files are merged in the listed order of preference:
user-defined values have higher priority than system-wide defaults

2
docs/topics/contracts.rst
View File

@ -64,7 +64,7 @@ Use the :command:`check` command to run the contract checks.
Custom Contracts
================
If you find you need more power than the built-in scrapy contracts you can
If you find you need more power than the built-in Scrapy contracts you can
create and load your own contracts in the project by using the
:setting:`SPIDER_CONTRACTS` setting::

2
docs/topics/debug.rst
View File

@ -5,7 +5,7 @@ Debugging Spiders
=================
This document explains the most common techniques for debugging spiders.
Consider the following scrapy spider below::
Consider the following Scrapy spider below::
import scrapy
from myproject.items import MyItem

40
docs/topics/developer-tools.rst
View File

@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things:
.. _topics-inspector:
Inspecting a website
===================================
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can
expand and collapse a tag by clicking on the arrow in front of it or by double
clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
copy XPaths to selected elements. Let's try it out: Right-click on the ``span``
tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
copy XPaths to selected elements. Let's try it out.
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
(...)
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
Then, back to your web browser, right-click on the ``span`` tag, select
``Copy > XPath`` and paste it in the Scrapy shell like so:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@ -112,13 +122,13 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_::
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
(...)]
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the scrapy shell::
we'll check another quite useful command from the Scrapy shell:
.. skip: next
.. code-block:: none
$ scrapy shell "quotes.toscrape.com/scroll"
(...)

28
docs/topics/dynamic-content.rst
View File

@ -172,27 +172,27 @@ data from it:
data in JSON format, which you can then parse with `json.loads`_.
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows::
``var data = {"field": "value"};`` you can extract that data as follows:
>>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
>>> json_data = response.css('script::text').re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
>>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
>>> json_data = response.css('script::text').re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
For example, if the JavaScript code contains
``var data = {field: "value"};`` you can extract that data as follows::
``var data = {field: "value"};`` you can extract that data as follows:
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css('script::text').get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css('script::text').get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. _topics-javascript-rendering:

106
docs/topics/items.rst
View File

@ -84,77 +84,74 @@ notice the API is very similar to the `dict API`_.
Creating items
--------------
::
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
Getting field values
--------------------
::
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
>>> product['price']
1000
>>> product['price']
1000
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product.get('last_updated', 'not set')
not set
>>> product.get('last_updated', 'not set')
not set
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product.get('lala', 'unknown field')
'unknown field'
>>> product.get('lala', 'unknown field')
'unknown field'
>>> 'name' in product # is name field populated?
True
>>> 'name' in product # is name field populated?
True
>>> 'last_updated' in product # is last_updated populated?
False
>>> 'last_updated' in product # is last_updated populated?
False
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> 'lala' in product.fields # is lala a declared field?
False
>>> 'lala' in product.fields # is lala a declared field?
False
Setting field values
--------------------
::
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
To access all populated values, just use the typical `dict API`_::
To access all populated values, just use the typical `dict API`_:
>>> product.keys()
['price', 'name']
>>> product.keys()
['price', 'name']
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
.. _copying-items:
@ -194,20 +191,21 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
Other common tasks
------------------
Creating dicts from items::
Creating dicts from items:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
Creating items from dicts::
Creating items from dicts:
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Extending Items
===============

2
docs/topics/jobs.rst
View File

@ -74,7 +74,7 @@ Request serialization
For persistence to work, :class:`~scrapy.http.Request` objects must be
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
values passed to their ``__init__`` method, which must be methods of the
runnning :class:`~scrapy.spiders.Spider` class.
running :class:`~scrapy.spiders.Spider` class.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.

142
docs/topics/leaks.rst
View File

@ -110,7 +110,7 @@ ties the response lifetime to the requests' one, and that would definitely
cause memory leaks.
Let's see how we can discover the cause (without knowing it
a-priori, of course) by using the ``trackref`` tool.
a priori, of course) by using the ``trackref`` tool.
After the crawler is running for a few minutes and we notice its memory usage
has grown a lot, we can enter its telnet console and check the live
@ -132,21 +132,21 @@ and check the code of the spider to discover the nasty line that is
generating the leaks (passing response references inside requests).
Sometimes extra information about live objects can be helpful.
Let's check the oldest response::
Let's check the oldest response:
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
If you want to iterate over all objects, instead of getting the oldest one, you
can use the :func:`scrapy.utils.trackref.iter_all` function::
can use the :func:`scrapy.utils.trackref.iter_all` function:
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
Too many spiders?
-----------------
@ -155,10 +155,10 @@ If your project has too many spiders executed in parallel,
the output of :func:`prefs()` can be difficult to read.
For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclases). For
example, this won't show any live references to spiders::
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. module:: scrapy.utils.trackref
:synopsis: Track references of live objects
@ -214,41 +214,41 @@ If you use ``pip``, you can install Guppy with the following command::
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
Guppy heap objects. Here's an example to view all Python objects available in
the heap using Guppy::
the heap using Guppy:
>>> x = hpy.heap()
>>> x.bytype
Partition of a set of 297033 objects. Total size = 52587824 bytes.
Index Count % Size % Cumulative % Type
0 22307 8 16423880 31 16423880 31 dict
1 122285 41 12441544 24 28865424 55 str
2 68346 23 5966696 11 34832120 66 tuple
3 227 0 5836528 11 40668648 77 unicode
4 2461 1 2222272 4 42890920 82 type
5 16870 6 2024400 4 44915320 85 function
6 13949 5 1673880 3 46589200 89 types.CodeType
7 13422 5 1653104 3 48242304 92 list
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
<1676 more rows. Type e.g. '_.more' to view.>
>>> x = hpy.heap()
>>> x.bytype
Partition of a set of 297033 objects. Total size = 52587824 bytes.
Index Count % Size % Cumulative % Type
0 22307 8 16423880 31 16423880 31 dict
1 122285 41 12441544 24 28865424 55 str
2 68346 23 5966696 11 34832120 66 tuple
3 227 0 5836528 11 40668648 77 unicode
4 2461 1 2222272 4 42890920 82 type
5 16870 6 2024400 4 44915320 85 function
6 13949 5 1673880 3 46589200 89 types.CodeType
7 13422 5 1653104 3 48242304 92 list
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
<1676 more rows. Type e.g. '_.more' to view.>
You can see that most space is used by dicts. Then, if you want to see from
which attribute those dicts are referenced, you could do::
which attribute those dicts are referenced, you could do:
>>> x.bytype[0].byvia
Partition of a set of 22307 objects. Total size = 16423880 bytes.
Index Count % Size % Cumulative % Referred Via:
0 10982 49 9416336 57 9416336 57 '.__dict__'
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
2 3097 14 1122904 7 13220744 80
3 990 4 277200 2 13497944 82 "['cookies']"
4 987 4 276360 2 13774304 84 "['cache']"
5 985 4 275800 2 14050104 86 "['meta']"
6 897 4 251160 2 14301264 87 '[2]'
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
9 27 0 155016 1 14841328 90 '[1]'
<333 more rows. Type e.g. '_.more' to view.>
>>> x.bytype[0].byvia
Partition of a set of 22307 objects. Total size = 16423880 bytes.
Index Count % Size % Cumulative % Referred Via:
0 10982 49 9416336 57 9416336 57 '.__dict__'
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
2 3097 14 1122904 7 13220744 80
3 990 4 277200 2 13497944 82 "['cookies']"
4 987 4 276360 2 13774304 84 "['cache']"
5 985 4 275800 2 14050104 86 "['meta']"
6 897 4 251160 2 14301264 87 '[2]'
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
9 27 0 155016 1 14841328 90 '[1]'
<333 more rows. Type e.g. '_.more' to view.>
As you can see, the Guppy module is very powerful but also requires some deep
knowledge about Python internals. For more info about Guppy, refer to the
@ -269,32 +269,32 @@ If you use ``pip``, you can install muppy with the following command::
pip install Pympler
Here's an example to view all Python objects available in
the heap using muppy::
the heap using muppy:
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
For more info about muppy, refer to the `muppy documentation`_.

45
docs/topics/link-extractors.rst
View File

@ -4,46 +4,33 @@
Link Extractors
===============
Link extractors are objects whose only purpose is to extract links from web
pages (:class:`scrapy.http.Response` objects) which will be eventually
followed.
A link extractor is an object that extracts links from responses.
There is ``scrapy.linkextractors.LinkExtractor`` available
in Scrapy, but you can create your own custom Link Extractors to suit your
needs by implementing a simple interface.
The only public method that every link extractor has is ``extract_links``,
which receives a :class:`~scrapy.http.Response` object and returns a list
of :class:`scrapy.link.Link` objects. Link extractors are meant to be
instantiated once and their ``extract_links`` method called several times
with different responses to extract links to follow.
Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider`
class (available in Scrapy), through a set of rules, but you can also use it in
your spiders, even if you don't subclass from
:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to
extract links.
The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
.. _topics-link-extractors-ref:
Built-in link extractors reference
==================================
Link extractor reference
========================
.. module:: scrapy.linkextractors
:synopsis: Link extractors classes
Link extractors classes bundled with Scrapy are provided in the
:mod:`scrapy.linkextractors` module.
The default link extractor is ``LinkExtractor``, which is the same as
:class:`~.LxmlLinkExtractor`::
The link extractor class is
:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
can also be imported as ``scrapy.linkextractors.LinkExtractor``::
from scrapy.linkextractors import LinkExtractor
There used to be other link extractor classes in previous Scrapy versions,
but they are deprecated now.
LxmlLinkExtractor
-----------------
@ -152,4 +139,6 @@ LxmlLinkExtractor
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
.. automethod:: extract_links
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

116
docs/topics/loaders.rst
View File

@ -206,14 +206,12 @@ metadata. Here is an example::
output_processor=TakeFirst(),
)
::
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
The precedence order, for both input and output processors, is as follows:
@ -314,11 +312,11 @@ ItemLoader objects
applied before processors
:type re: str or compiled regex
Examples::
Examples:
>>> from scrapy.loader.processors import TakeFirst
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
>>> from scrapy.loader.processors import TakeFirst
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
.. method:: add_value(field_name, value, \*processors, \**kwargs)
@ -477,6 +475,8 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
This is mostly used as a property so when attempting to override this
value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
@ -637,12 +637,12 @@ Here is a list of all built-in processors:
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
Example::
Example:
>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['one', 'two', 'three'])
['one', 'two', 'three']
>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['one', 'two', 'three'])
['one', 'two', 'three']
.. class:: TakeFirst
@ -650,12 +650,12 @@ Here is a list of all built-in processors:
so it's typically used as an output processor to single-valued fields.
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
Example::
Example:
>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'one', 'two', 'three'])
'one'
>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'one', 'two', 'three'])
'one'
.. class:: Join(separator=u' ')
@ -665,15 +665,15 @@ Here is a list of all built-in processors:
When using the default separator, this processor is equivalent to the
function: ``u' '.join``
Examples::
Examples:
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
'one two three'
>>> proc = Join('<br>')
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
'one two three'
>>> proc = Join('<br>')
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
.. class:: Compose(\*functions, \**default_loader_context)
@ -686,12 +686,12 @@ Here is a list of all built-in processors:
By default, stop process on ``None`` value. This behaviour can be changed by
passing keyword argument ``stop_on_none=False``.
Example::
Example:
>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['hello', 'world'])
'HELLO'
>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['hello', 'world'])
'HELLO'
Each function can optionally receive a ``loader_context`` parameter. For
those which do, this processor will pass the currently active :ref:`Loader
@ -730,15 +730,15 @@ Here is a list of all built-in processors:
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
<topics-selectors>`, which returns a list of unicode strings.
The example below should clarify how it works::
The example below should clarify how it works:
>>> def filter_world(x):
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
>>> proc = MapCompose(filter_world, str.upper)
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
['HELLO, 'THIS', 'IS', 'SCRAPY']
>>> def filter_world(x):
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
>>> proc = MapCompose(filter_world, str.upper)
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
``__init__`` method keyword arguments are used as default context values. See
@ -750,21 +750,21 @@ Here is a list of all built-in processors:
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
Example::
Example:
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
>>> proc({'foo': 'bar'})
'bar'
>>> proc({'foo': {'bar': 'baz'}})
{'bar': 'baz'}
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
>>> proc({'foo': 'bar'})
'bar'
>>> proc({'foo': {'bar': 'baz'}})
{'bar': 'baz'}
Working with Json::
Working with Json:
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
['bar']
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
['bar']

4
docs/topics/logging.rst
View File

@ -171,9 +171,9 @@ listed in `logging's logrecord attributes docs
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
respectively.
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
scrapy component responsible for that log output.
Scrapy component responsible for that log output.
Command-line options
--------------------

2
docs/topics/media-pipeline.rst
View File

@ -558,7 +558,7 @@ See here the methods that you can override in your custom Images Pipeline:
Custom Images pipeline example
==============================
Here is a full example of the Images Pipeline whose methods are examplified
Here is a full example of the Images Pipeline whose methods are exemplified
above::
import scrapy

565
docs/topics/selectors.rst
View File

@ -51,18 +51,18 @@ Constructing selectors
.. highlight:: python
Response objects expose a :class:`~scrapy.selector.Selector` instance
on ``.selector`` attribute::
on ``.selector`` attribute:
>>> response.selector.xpath('//span/text()').get()
'good'
>>> response.selector.xpath('//span/text()').get()
'good'
Querying responses using XPath and CSS is so common that responses include two
more shortcuts: ``response.xpath()`` and ``response.css()``::
more shortcuts: ``response.xpath()`` and ``response.css()``:
>>> response.xpath('//span/text()').get()
'good'
>>> response.css('span::text').get()
'good'
>>> response.xpath('//span/text()').get()
'good'
>>> response.css('span::text').get()
'good'
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
@ -74,21 +74,21 @@ shortcuts. By using ``response.selector`` or one of these shortcuts
you can also ensure the response body is parsed only once.
But if required, it is possible to use ``Selector`` directly.
Constructing from text::
Constructing from text:
>>> from scrapy.selector import Selector
>>> body = '<html><body><span>good</span></body></html>'
>>> Selector(text=body).xpath('//span/text()').get()
'good'
>>> from scrapy.selector import Selector
>>> body = '<html><body><span>good</span></body></html>'
>>> Selector(text=body).xpath('//span/text()').get()
'good'
Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
:class:`~scrapy.http.TextResponse` subclasses::
:class:`~scrapy.http.TextResponse` subclasses:
>>> from scrapy.selector import Selector
>>> from scrapy.http import HtmlResponse
>>> response = HtmlResponse(url='http://example.com', body=body)
>>> Selector(response=response).xpath('//span/text()').get()
'good'
>>> from scrapy.selector import Selector
>>> from scrapy.http import HtmlResponse
>>> response = HtmlResponse(url='http://example.com', body=body)
>>> Selector(response=response).xpath('//span/text()').get()
'good'
``Selector`` automatically chooses the best parsing rules
(XML vs HTML) based on input type.
@ -123,118 +123,118 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
.. highlight:: python
So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
page, let's construct an XPath for selecting the text inside the title tag::
page, let's construct an XPath for selecting the text inside the title tag:
>>> response.xpath('//title/text()')
[<Selector xpath='//title/text()' data='Example website'>]
>>> response.xpath('//title/text()')
[<Selector xpath='//title/text()' data='Example website'>]
To actually extract the textual data, you must call the selector ``.get()``
or ``.getall()`` methods, as follows::
or ``.getall()`` methods, as follows:
>>> response.xpath('//title/text()').getall()
['Example website']
>>> response.xpath('//title/text()').get()
'Example website'
>>> response.xpath('//title/text()').getall()
['Example website']
>>> response.xpath('//title/text()').get()
'Example website'
``.get()`` always returns a single result; if there are several matches,
content of a first match is returned; if there are no matches, None
is returned. ``.getall()`` returns a list with all results.
Notice that CSS selectors can select text or attribute nodes using CSS3
pseudo-elements::
pseudo-elements:
>>> response.css('title::text').get()
'Example website'
>>> response.css('title::text').get()
'Example website'
As you can see, ``.xpath()`` and ``.css()`` methods return a
:class:`~scrapy.selector.SelectorList` instance, which is a list of new
selectors. This API can be used for quickly selecting nested data::
selectors. This API can be used for quickly selecting nested data:
>>> response.css('img').xpath('@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.css('img').xpath('@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
If you want to extract only the first matched element, you can call the
selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
previous Scrapy versions)::
previous Scrapy versions):
>>> response.xpath('//div[@id="images"]/a/text()').get()
'Name: My image 1 '
>>> response.xpath('//div[@id="images"]/a/text()').get()
'Name: My image 1 '
It returns ``None`` if no element was found::
It returns ``None`` if no element was found:
>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
True
>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
True
A default return value can be provided as an argument, to be used instead
of ``None``:
>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
'not-found'
>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
'not-found'
Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
>>> [img.attrib['src'] for img in response.css('img')]
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> [img.attrib['src'] for img in response.css('img')]
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
As a shortcut, ``.attrib`` is also available on SelectorList directly;
it returns attributes for the first matching element::
it returns attributes for the first matching element:
>>> response.css('img').attrib['src']
'image1_thumb.jpg'
>>> response.css('img').attrib['src']
'image1_thumb.jpg'
This is most useful when only a single result is expected, e.g. when selecting
by id, or selecting unique elements on a web page::
by id, or selecting unique elements on a web page:
>>> response.css('base').attrib['href']
'http://example.com/'
>>> response.css('base').attrib['href']
'http://example.com/'
Now we're going to get the base URL and some image links::
Now we're going to get the base URL and some image links:
>>> response.xpath('//base/@href').get()
'http://example.com/'
>>> response.xpath('//base/@href').get()
'http://example.com/'
>>> response.css('base::attr(href)').get()
'http://example.com/'
>>> response.css('base::attr(href)').get()
'http://example.com/'
>>> response.css('base').attrib['href']
'http://example.com/'
>>> response.css('base').attrib['href']
'http://example.com/'
>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.css('a[href*=image]::attr(href)').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.css('a[href*=image]::attr(href)').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.css('a[href*=image] img::attr(src)').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.css('a[href*=image] img::attr(src)').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
.. _topics-selectors-css-extensions:
@ -259,47 +259,47 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
Examples:
* ``title::text`` selects children text nodes of a descendant ``<title>`` element::
* ``title::text`` selects children text nodes of a descendant ``<title>`` element:
>>> response.css('title::text').get()
'Example website'
>>> response.css('title::text').get()
'Example website'
* ``*::text`` selects all descendant text nodes of the current selector context::
* ``*::text`` selects all descendant text nodes of the current selector context:
>>> response.css('#images *::text').getall()
['\n ',
'Name: My image 1 ',
'\n ',
'Name: My image 2 ',
'\n ',
'Name: My image 3 ',
'\n ',
'Name: My image 4 ',
'\n ',
'Name: My image 5 ',
'\n ']
>>> response.css('#images *::text').getall()
['\n ',
'Name: My image 1 ',
'\n ',
'Name: My image 2 ',
'\n ',
'Name: My image 3 ',
'\n ',
'Name: My image 4 ',
'\n ',
'Name: My image 5 ',
'\n ']
* ``foo::text`` returns no results if ``foo`` element exists, but contains
no text (i.e. text is empty)::
no text (i.e. text is empty):
>>> response.css('img::text').getall()
[]
>>> response.css('img::text').getall()
[]
This means ``.css('foo::text').get()`` could return None even if an element
exists. Use ``default=''`` if you always want a string::
exists. Use ``default=''`` if you always want a string:
>>> response.css('img::text').get()
>>> response.css('img::text').get(default='')
''
>>> response.css('img::text').get()
>>> response.css('img::text').get(default='')
''
* ``a::attr(href)`` selects the *href* attribute value of descendant links::
* ``a::attr(href)`` selects the *href* attribute value of descendant links:
>>> response.css('a::attr(href)').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.css('a::attr(href)').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
.. note::
See also: :ref:`selecting-attributes`.
@ -318,25 +318,24 @@ Nesting selectors
The selection methods (``.xpath()`` or ``.css()``) return a list of selectors
of the same type, so you can call the selection methods for those selectors
too. Here's an example::
too. Here's an example:
>>> links = response.xpath('//a[contains(@href, "image")]')
>>> links.getall()
['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> links = response.xpath('//a[contains(@href, "image")]')
>>> links.getall()
['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
... print('Link number %d points to url %r and image %r' % args)
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
>>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
... print('Link number %d points to url %r and image %r' % args)
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
.. _selecting-attributes:
@ -344,42 +343,42 @@ Selecting element attributes
----------------------------
There are several ways to get a value of an attribute. First, one can use
XPath syntax::
XPath syntax:
>>> response.xpath("//a/@href").getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> response.xpath("//a/@href").getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
XPath syntax has a few advantages: it is a standard XPath feature, and
``@attributes`` can be used in other parts of an XPath expression - e.g.
it is possible to filter by attribute value.
Scrapy also provides an extension to CSS selectors (``::attr(...)``)
which allows to get attribute values::
which allows to get attribute values:
>>> response.css('a::attr(href)').getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> response.css('a::attr(href)').getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
In addition to that, there is a ``.attrib`` property of Selector.
You can use it if you prefer to lookup attributes in Python
code, without using XPaths or CSS extensions::
code, without using XPaths or CSS extensions:
>>> [a.attrib['href'] for a in response.css('a')]
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> [a.attrib['href'] for a in response.css('a')]
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
This property is also available on SelectorList; it returns a dictionary
with attributes of a first matching element. It is convenient to use when
a selector is expected to give a single result (e.g. when selecting by element
ID, or when selecting an unique element on a page)::
ID, or when selecting an unique element on a page):
>>> response.css('base').attrib
{'href': 'http://example.com/'}
>>> response.css('base').attrib['href']
'http://example.com/'
>>> response.css('base').attrib
{'href': 'http://example.com/'}
>>> response.css('base').attrib['href']
'http://example.com/'
``.attrib`` property of an empty SelectorList is empty::
``.attrib`` property of an empty SelectorList is empty:
>>> response.css('foo').attrib
{}
>>> response.css('foo').attrib
{}
Using selectors with regular expressions
----------------------------------------
@ -390,21 +389,21 @@ data using regular expressions. However, unlike using ``.xpath()`` or
can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
<topics-selectors-htmlcode>` above::
<topics-selectors-htmlcode>` above:
>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
['My image 1',
'My image 2',
'My image 3',
'My image 4',
'My image 5']
>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
['My image 1',
'My image 2',
'My image 3',
'My image 4',
'My image 5']
There's an additional helper reciprocating ``.get()`` (and its
alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
Use it to extract just the first matching string::
Use it to extract just the first matching string:
>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
'My image 1'
>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
'My image 1'
.. _old-extraction-api:
@ -422,28 +421,28 @@ and readable code.
The following examples show how these methods map to each other.
1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:
>>> response.css('a::attr(href)').get()
'image1.html'
>>> response.css('a::attr(href)').extract_first()
'image1.html'
>>> response.css('a::attr(href)').get()
'image1.html'
>>> response.css('a::attr(href)').extract_first()
'image1.html'
2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:
>>> response.css('a::attr(href)').getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> response.css('a::attr(href)').extract()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> response.css('a::attr(href)').getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> response.css('a::attr(href)').extract()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
3. ``Selector.get()`` is the same as ``Selector.extract()``::
3. ``Selector.get()`` is the same as ``Selector.extract()``:
>>> response.css('a::attr(href)')[0].get()
'image1.html'
>>> response.css('a::attr(href)')[0].extract()
'image1.html'
>>> response.css('a::attr(href)')[0].get()
'image1.html'
>>> response.css('a::attr(href)')[0].extract()
'image1.html'
4. For consistency, there is also ``Selector.getall()``, which returns a list::
4. For consistency, there is also ``Selector.getall()``, which returns a list:
>>> response.css('a::attr(href)')[0].getall()
['image1.html']
@ -481,26 +480,26 @@ with ``/``, that XPath will be absolute to the document and not relative to the
``Selector`` you're calling it from.
For example, suppose you want to extract all ``<p>`` elements inside ``<div>``
elements. First, you would get all ``<div>`` elements::
elements. First, you would get all ``<div>`` elements:
>>> divs = response.xpath('//div')
>>> divs = response.xpath('//div')
At first, you may be tempted to use the following approach, which is wrong, as
it actually extracts all ``<p>`` elements from the document, not only those
inside ``<div>`` elements::
inside ``<div>`` elements:
>>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document
... print(p.get())
>>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document
... print(p.get())
This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):
>>> for p in divs.xpath('.//p'): # extracts all <p> inside
... print(p.get())
>>> for p in divs.xpath('.//p'): # extracts all <p> inside
... print(p.get())
Another common case would be to extract all direct ``<p>`` children::
Another common case would be to extract all direct ``<p>`` children:
>>> for p in divs.xpath('p'):
... print(p.get())
>>> for p in divs.xpath('p'):
... print(p.get())
For more details about relative XPaths see the `Location Paths`_ section in the
XPath specification.
@ -521,12 +520,12 @@ for that you may end up with more elements that you want, if they have a differe
class name that shares the string ``someclass``.
As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
you can just select by class using CSS and then switch to XPath when needed::
you can just select by class using CSS and then switch to XPath when needed:
>>> from scrapy import Selector
>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
>>> sel.css('.shout').xpath('./time/@datetime').getall()
['2014-07-23 19:00']
>>> from scrapy import Selector
>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
>>> sel.css('.shout').xpath('./time/@datetime').getall()
['2014-07-23 19:00']
This is cleaner than using the verbose XPath trick shown above. Just remember
to use the ``.`` in the XPath expressions that will follow.
@ -538,41 +537,41 @@ Beware of the difference between //node[1] and (//node)[1]
``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
Example::
Example:
>>> from scrapy import Selector
>>> sel = Selector(text="""
....: <ul class="list">
....: <li>1</li>
....: <li>2</li>
....: <li>3</li>
....: </ul>
....: <ul class="list">
....: <li>4</li>
....: <li>5</li>
....: <li>6</li>
....: </ul>""")
>>> xp = lambda x: sel.xpath(x).getall()
>>> from scrapy import Selector
>>> sel = Selector(text="""
....: <ul class="list">
....: <li>1</li>
....: <li>2</li>
....: <li>3</li>
....: </ul>
....: <ul class="list">
....: <li>4</li>
....: <li>5</li>
....: <li>6</li>
....: </ul>""")
>>> xp = lambda x: sel.xpath(x).getall()
This gets all first ``<li>`` elements under whatever it is its parent::
This gets all first ``<li>`` elements under whatever it is its parent:
>>> xp("//li[1]")
['<li>1</li>', '<li>4</li>']
>>> xp("//li[1]")
['<li>1</li>', '<li>4</li>']
And this gets the first ``<li>`` element in the whole document::
And this gets the first ``<li>`` element in the whole document:
>>> xp("(//li)[1]")
['<li>1</li>']
>>> xp("(//li)[1]")
['<li>1</li>']
This gets all first ``<li>`` elements under an ``<ul>`` parent::
This gets all first ``<li>`` elements under an ``<ul>`` parent:
>>> xp("//ul/li[1]")
['<li>1</li>', '<li>4</li>']
>>> xp("//ul/li[1]")
['<li>1</li>', '<li>4</li>']
And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document::
And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document:
>>> xp("(//ul/li)[1]")
['<li>1</li>']
>>> xp("(//ul/li)[1]")
['<li>1</li>']
Using text nodes in a condition
-------------------------------
@ -584,34 +583,34 @@ This is because the expression ``.//text()`` yields a collection of text element
And when a node-set is converted to a string, which happens when it is passed as argument to
a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
Example::
Example:
>>> from scrapy import Selector
>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
>>> from scrapy import Selector
>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
Converting a *node-set* to string::
Converting a *node-set* to string:
>>> sel.xpath('//a//text()').getall() # take a peek at the node-set
['Click here to go to the ', 'Next Page']
>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
['Click here to go to the ']
>>> sel.xpath('//a//text()').getall() # take a peek at the node-set
['Click here to go to the ', 'Next Page']
>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
['Click here to go to the ']
A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
A *node* converted to a string, however, puts together the text of itself plus of all its descendants:
>>> sel.xpath("//a[1]").getall() # select the first node
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
>>> sel.xpath("string(//a[1])").getall() # convert it to string
['Click here to go to the Next Page']
>>> sel.xpath("//a[1]").getall() # select the first node
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
>>> sel.xpath("string(//a[1])").getall() # convert it to string
['Click here to go to the Next Page']
So, using the ``.//text()`` node-set won't select anything in this case::
So, using the ``.//text()`` node-set won't select anything in this case:
>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
[]
>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
[]
But using the ``.`` to mean the node, works::
But using the ``.`` to mean the node, works:
>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
@ -627,17 +626,17 @@ some arguments in your queries with placeholders like ``?``,
which are then substituted with values passed with the query.
Here's an example to match an element based on its "id" attribute value,
without hard-coding it (that was shown previously)::
without hard-coding it (that was shown previously):
>>> # `$val` used in the expression, a `val` argument needs to be passed
>>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
'Name: My image 1 '
>>> # `$val` used in the expression, a `val` argument needs to be passed
>>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
'Name: My image 1 '
Here's another example, to find the "id" attribute of a ``<div>`` tag containing
five ``<a>`` children (here we pass the value ``5`` as an integer)::
five ``<a>`` children (here we pass the value ``5`` as an integer):
>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
'images'
>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
'images'
All variable references must have a binding value when calling ``.xpath()``
(otherwise you'll get a ``ValueError: XPath error:`` exception).
@ -687,19 +686,19 @@ You can see several namespace declarations including a default
.. highlight:: python
Once in the shell we can try selecting all ``<link>`` objects and see that it
doesn't work (because the Atom XML namespace is obfuscating those nodes)::
doesn't work (because the Atom XML namespace is obfuscating those nodes):
>>> response.xpath("//link")
[]
>>> response.xpath("//link")
[]
But once we call the :meth:`Selector.remove_namespaces` method, all
nodes can be accessed directly by their names::
nodes can be accessed directly by their names:
>>> response.selector.remove_namespaces()
>>> response.xpath("//link")
[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
<Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
...
>>> response.selector.remove_namespaces()
>>> response.xpath("//link")
[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
<Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
...
If you wonder why the namespace removal procedure isn't always called by default
instead of having to call it manually, this is because of two reasons, which, in order
@ -734,26 +733,25 @@ Regular expressions
The ``test()`` function, for example, can prove quite useful when XPath's
``starts-with()`` or ``contains()`` are not sufficient.
Example selecting links in list item with a "class" attribute ending with a digit::
Example selecting links in list item with a "class" attribute ending with a digit:
>>> from scrapy import Selector
>>> doc = u"""
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
... <li class="item-1"><a href="link2.html">second item</a></li>
... <li class="item-inactive"><a href="link3.html">third item</a></li>
... <li class="item-1"><a href="link4.html">fourth item</a></li>
... <li class="item-0"><a href="link5.html">fifth item</a></li>
... </ul>
... </div>
... """
>>> sel = Selector(text=doc, type="html")
>>> sel.xpath('//li//@href').getall()
['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
['link1.html', 'link2.html', 'link4.html', 'link5.html']
>>>
>>> from scrapy import Selector
>>> doc = u"""
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
... <li class="item-1"><a href="link2.html">second item</a></li>
... <li class="item-inactive"><a href="link3.html">third item</a></li>
... <li class="item-1"><a href="link4.html">fourth item</a></li>
... <li class="item-0"><a href="link5.html">fifth item</a></li>
... </ul>
... </div>
... """
>>> sel = Selector(text=doc, type="html")
>>> sel.xpath('//li//@href').getall()
['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
['link1.html', 'link2.html', 'link4.html', 'link5.html']
.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
@ -849,7 +847,6 @@ with groups of itemscopes and corresponding itemprops::
current scope: ['http://schema.org/Rating']
properties: ['worstRating', 'ratingValue', 'bestRating']
>>>
Here we first iterate over ``itemscope`` elements, and for each one,
we look for all ``itemprops`` elements and exclude those that are themselves
@ -877,15 +874,15 @@ For the following HTML::
.. highlight:: python
You can use it like this::
You can use it like this:
>>> response.xpath('//p[has-class("foo")]')
[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
<Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
>>> response.xpath('//p[has-class("foo", "bar-baz")]')
[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
>>> response.xpath('//p[has-class("foo", "bar")]')
[]
>>> response.xpath('//p[has-class("foo")]')
[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
<Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
>>> response.xpath('//p[has-class("foo", "bar-baz")]')
[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
>>> response.xpath('//p[has-class("foo", "bar")]')
[]
So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
``p.foo.bar-baz``. Please note, that it is slower in most of the cases,

6
docs/topics/settings.rst
View File

@ -884,7 +884,7 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available
String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
placeholders.
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
@ -1264,7 +1264,7 @@ Default::
'scrapy.contracts.default.ScrapesContract': 3,
}
A dict containing the scrapy contracts enabled by default in Scrapy. You should
A dict containing the Scrapy contracts enabled by default in Scrapy. You should
never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
instead. For more info see :ref:`topics-contracts`.
@ -1295,7 +1295,7 @@ SPIDER_LOADER_WARN_ONLY
Default: ``False``
By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
it will fail loudly if there is any ``ImportError`` exception.
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.

79
docs/topics/shell.rst
View File

@ -31,7 +31,7 @@ for more info.
Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
is unavailable.
Through scrapy's settings you can configure it to use any one of
Through Scrapy's settings you can configure it to use any one of
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
@ -177,47 +177,46 @@ all start with the ``[s]`` prefix)::
>>>
After that, we can start playing with the objects::
After that, we can start playing with the objects:
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> fetch("https://reddit.com")
>>> fetch("https://reddit.com")
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'
>>> request = request.replace(method="POST")
>>> request = request.replace(method="POST")
>>> fetch(request)
>>> fetch(request)
>>> response.status
404
>>> response.status
404
>>> from pprint import pprint
>>> from pprint import pprint
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
>>>
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
.. _topics-shell-inspect-response:
@ -263,16 +262,16 @@ When you run the spider, you will get something similar to this::
>>> response.url
'http://example.org'
Then, you can check if the extraction code is working::
Then, you can check if the extraction code is working:
>>> response.xpath('//h1[@class="fn"]')
[]
>>> response.xpath('//h1[@class="fn"]')
[]
Nope, it doesn't. So you can open the response in your web browser and see if
it's the response you were expecting::
it's the response you were expecting:
>>> view(response)
True
>>> view(response)
True
Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
crawling::

6
docs/topics/spiders.rst
View File

@ -414,6 +414,12 @@ Crawling rules
from which the request originated as second argument. It must return a
``Request`` object or ``None`` (to filter out the request).
``errback`` is a callable or a string (in which case a method from the spider
object with that name will be used) to be called if any exception is
raised while processing a request generated by the rule.
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~

12
docs/topics/stats.rst
View File

@ -57,15 +57,15 @@ Set stat value only if lower than previous::
stats.min_value('min_free_memory_percent', value)
Get stat value::
Get stat value:
>>> stats.get_value('custom_count')
1
>>> stats.get_value('custom_count')
1
Get all stats::
Get all stats:
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================

4
docs/topics/telnetconsole.rst
View File

@ -44,11 +44,11 @@ the console you need to type::
>>>
By default Username is ``scrapy`` and Password is autogenerated. The
autogenerated Password can be seen on scrapy logs like the example below::
autogenerated Password can be seen on Scrapy logs like the example below::
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
Default Username and Password can be overriden by the settings
Default Username and Password can be overridden by the settings
:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
.. warning::

1
extras/qps-bench-server.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python
from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET

91
pytest.ini
View File

@ -8,7 +8,6 @@ addopts =
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
--ignore=docs/topics/developer-tools.rst
--ignore=docs/topics/dynamic-content.rst
--ignore=docs/topics/items.rst
--ignore=docs/topics/leaks.rst
@ -26,52 +25,52 @@ flake8-ignore =
scrapy/http/__init__.py F401
# Issues pending a review:
# extras
extras/qps-bench-server.py E261 E501
extras/qpsclient.py E501 E261 E501
extras/qps-bench-server.py E501
extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E502 E731
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731 E226
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/startproject.py E502 E127 E501 E128
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E502 E128
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
scrapy/core/engine.py E501 E128 E127 E306 E502
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E306 E261 E128 W504
scrapy/core/spidermw.py E501 E731 E502 E126 E226
scrapy/core/scraper.py E501 E306 E128 W504
scrapy/core/spidermw.py E501 E731 E126 E226
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E305 E241
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E502 E128
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
scrapy/extensions/closespider.py E501 E502 E128 E123
scrapy/extensions/closespider.py E501 E128 E123
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501 E303
@ -85,13 +84,13 @@ flake8-ignore =
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128 W293 W291
scrapy/http/response/text.py E501 W293 E128 E124
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E502 E501 E402
scrapy/linkextractors/__init__.py E731 E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
# scrapy/loader
scrapy/loader/__init__.py E501 E502 E128
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/files.py E116 E501 E266
@ -102,7 +101,7 @@ flake8-ignore =
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
scrapy/settings/default_settings.py E501 E114 E116 E226
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
@ -112,25 +111,24 @@ flake8-ignore =
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
scrapy/spiders/feed.py E501 E261
scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/benchserver.py E501
scrapy/utils/conf.py E402 E502 E501
scrapy/utils/console.py E261 E306 E305
scrapy/utils/conf.py E402 E501
scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/engine.py E261
scrapy/utils/gz.py E305 E501 W504
scrapy/utils/http.py F403 E226
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/log.py E128 W503
scrapy/utils/markup.py F403 W292
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501 E226
scrapy/utils/multipart.py F403 W292
scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E226
@ -145,18 +143,17 @@ flake8-ignore =
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/_monkeypatches.py W293
scrapy/cmdline.py E502 E501
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501 E261 E226
scrapy/exporters.py E501 E226
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501 W293
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E502 E128 E501
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/responsetypes.py E128 E501 E305
scrapy/robotstxt.py E501
@ -171,15 +168,15 @@ flake8-ignore =
tests/pipelines.py F841 E226
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_fetch.py E501 E261
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128 W293
tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py F841 E501 E305
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
@ -190,18 +187,18 @@ flake8-ignore =
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
tests/test_engine.py E401 E501 E502 E128 E261
tests/test_dupefilters.py E221 E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
tests/test_http_response.py E501 E301 E502 E128 E265
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E301 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
@ -210,28 +207,28 @@ flake8-ignore =
tests/test_mail.py E128 E501 E305
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
tests/test_pipeline_files.py E501 W293 E303 E272 E226
tests/test_pipeline_files.py E501 E303 E272 E226
tests/test_pipeline_images.py F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E305
tests/test_robotstxt_interface.py E501 W291 E501
tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
tests/test_spidermiddleware_output_chain.py E501 W293 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
tests/test_utils_defer.py E306 E261 E501 F841 E226
tests/test_utils_defer.py E306 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
tests/test_utils_http.py E501 E502 E128 W504
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E303 E241
tests/test_utils_log.py E741 E226
tests/test_utils_python.py E501 E303 E731 E701 E305
@ -240,11 +237,11 @@ flake8-ignore =
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731 E226
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_spider.py E261 E305
tests/test_utils_spider.py E305
tests/test_utils_template.py E305
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
tests/test_cmdline/__init__.py E502 E501
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501

18
requirements-py3.txt
View File

@ -1,18 +0,0 @@
parsel>=1.5.0
PyDispatcher>=2.0.5
Twisted>=17.9.0
w3lib>=1.17.0
protego>=0.1.15
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
cryptography>=2.0 # Earlier versions would fail to install
# Reference versions taken from
# https://packages.ubuntu.com/xenial/python/
# https://packages.ubuntu.com/xenial/zope/
cssselect>=0.9.1
lxml>=3.5.0
service_identity>=16.0.0
six>=1.10.0
zope.interface>=4.1.3

2
scrapy/__init__.py
View File

@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
from . import _monkeypatches
from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv

2
scrapy/_monkeypatches.py
View File

@ -1,4 +1,4 @@
from six.moves import copyreg
import copyreg
# Undo what Twisted's perspective broker adds to pickle register

5
scrapy/cmdline.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import sys
import os
import optparse
@ -68,7 +67,7 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
@ -124,7 +123,7 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)

3
scrapy/commands/bench.py
View File

@ -1,8 +1,7 @@
import sys
import time
import subprocess
from six.moves.urllib.parse import urlencode
from urllib.parse import urlencode
import scrapy
from scrapy.commands import ScrapyCommand

12
scrapy/commands/fetch.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import sys
from w3lib.url import is_url
@ -25,12 +24,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true", \
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():

1
scrapy/commands/genspider.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import os
import shutil
import string

1
scrapy/commands/list.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
from scrapy.commands import ScrapyCommand

1
scrapy/commands/parse.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import json
import logging

1
scrapy/commands/settings.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import json
from scrapy.commands import ScrapyCommand

5
scrapy/commands/startproject.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import re
import os
import string
@ -44,8 +43,8 @@ class Command(ScrapyCommand):
return False
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print('Error: Project names must begin with a letter and contain'\
' only\nletters, numbers and underscores')
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
else:

2
scrapy/commands/version.py
View File

@ -1,5 +1,3 @@
from __future__ import print_function
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions

6
scrapy/contracts/default.py
View File

@ -4,7 +4,7 @@ from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
from . import Contract
from scrapy.contracts import Contract
# contracts
@ -86,8 +86,8 @@ class ReturnsContract(Contract):
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
raise ContractFail("Returned %s %s, expected %s" % \
(occurrences, self.obj_name, expected))
raise ContractFail("Returned %s %s, expected %s" %
(occurrences, self.obj_name, expected))
class ScrapesContract(Contract):

7
scrapy/core/downloader/__init__.py
View File

@ -3,15 +3,14 @@ from time import time
from datetime import datetime
from collections import deque
import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
from .middleware import DownloaderMiddlewareManager
from .handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
@ -188,7 +187,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
for slot in six.itervalues(self.slots):
for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):

5
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,8 +1,9 @@
"""Download handlers for different schemes"""
import logging
from twisted.internet import defer
import six
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
@ -22,7 +23,7 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
for scheme, clspath in six.iteritems(handlers):
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)

2
scrapy/core/downloader/handlers/ftp.py
View File

@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
from six.moves.urllib.parse import unquote
from urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed

6
scrapy/core/downloader/handlers/http.py
View File

@ -1,2 +1,4 @@
from .http10 import HTTP10DownloadHandler
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import (
HTTP11DownloadHandler as HTTPDownloadHandler,
)

11
scrapy/core/downloader/handlers/http11.py
View File

@ -2,10 +2,10 @@
import re
import logging
import warnings
from io import BytesIO
from time import time
import warnings
from six.moves.urllib.parse import urldefrag
from urllib.parse import urldefrag
from zope.interface import implementer
from twisted.internet import defer, reactor, protocol
@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
@ -291,6 +292,12 @@ class ScrapyAgent(object):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Crawlera, it doesn't require this mode anymore, "
"so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.",
ScrapyDeprecationWarning)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)

6
scrapy/core/downloader/handlers/s3.py
View File

@ -1,9 +1,9 @@
from six.moves.urllib.parse import unquote
from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.boto import is_botocore
from .http import HTTPDownloadHandler
from scrapy.utils.httpobj import urlparse_cached
def _get_boto_connection():

8
scrapy/core/downloader/middleware.py
View File

@ -3,8 +3,6 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
import six
from twisted.internet import defer
from scrapy.exceptions import _InvalidOutput
@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
(six.get_method_self(method).__class__.__name__, response.__class__.__name__))
(method.__self__.__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, response=response, spider=spider)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
(six.get_method_self(method).__class__.__name__, type(response)))
(method.__self__.__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, exception=exception, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
(six.get_method_self(method).__class__.__name__, type(response)))
(method.__self__.__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)

10
scrapy/core/downloader/webclient.py
View File

@ -1,5 +1,5 @@
from time import time
from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
def connectionMade(self):
self.headers = Headers() # bucket for response headers
self.headers = Headers() # bucket for response headers
# Method command
self.sendCommand(self.factory.method, self.factory.path)
@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
if self.factory.url.startswith(b'https'):
self.transport.stopProducing()
self.factory.noPage(\
defer.TimeoutError("Getting %s took longer than %s seconds." % \
(self.factory.url, self.factory.timeout)))
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds." %
(self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):

2
scrapy/core/engine.py
View File

@ -25,7 +25,7 @@ class Slot(object):
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
self.inprogress = set() # requests in progress
self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall

19
scrapy/core/scraper.py
View File

@ -9,7 +9,7 @@ from twisted.internet import defer
from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.misc import load_object
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy import signals
@ -18,6 +18,7 @@ from scrapy.item import BaseItem
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.utils.request import referer_str
logger = logging.getLogger(__name__)
@ -99,11 +100,13 @@ class Scraper(object):
def enqueue_scrape(self, response, request, spider):
slot = self.slot
dfd = slot.add_response_request(response, request)
def finish_scraping(_):
slot.finish_response(response, request)
self._check_if_closing(spider, slot)
self._scrape_next(spider, slot)
return _
dfd.addBoth(finish_scraping)
dfd.addErrback(
lambda f: logger.error('Scraper bug processing %(request)s',
@ -123,7 +126,7 @@ class Scraper(object):
callback/errback"""
assert isinstance(response, (Response, Failure))
dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
return dfd
@ -142,7 +145,10 @@ class Scraper(object):
def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
dfd.addCallbacks(callback=request.callback or spider.parse,
callback = request.callback or spider.parse
warn_on_generator_with_return_value(spider, callback)
warn_on_generator_with_return_value(spider, request.errback)
dfd.addCallbacks(callback=callback,
errback=request.errback,
callbackKeywords=request.cb_kwargs)
return dfd.addCallback(iterate_spider_output)
@ -172,8 +178,8 @@ class Scraper(object):
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel(it, self.concurrent_items,
self._process_spidermw_output, request, response, spider)
dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
return dfd
def _process_spidermw_output(self, output, request, response, spider):
@ -200,8 +206,7 @@ class Scraper(object):
"""Log and silence errors that come from the engine (typically download
errors that got propagated thru here)
"""
if (isinstance(download_failure, Failure) and
not download_failure.check(IgnoreRequest)):
if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
if download_failure.frames:
logger.error('Error downloading %(request)s',
{'request': request},

11
scrapy/core/spidermw.py
View File

@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
"""
from itertools import chain, islice
import six
from twisted.python.failure import Failure
from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
@ -37,15 +36,15 @@ class SpiderMiddlewareManager(MiddlewareManager):
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f: '%s.%s' % (
six.get_method_self(f).__class__.__name__,
six.get_method_function(f).__name__)
f.__self__.__class__.__name__,
f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
if result is not None:
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
.format(fname(method), type(result)))
except _InvalidOutput:
raise
@ -70,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
.format(fname(method), type(result)))
return _failure
@ -104,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = evaluate_iterable(result, method_index)
else:
raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
raise _InvalidOutput('Middleware {} must return an iterable, got {}'
.format(fname(method), type(result)))
return chain(result, recovered)

8
scrapy/crawler.py
View File

@ -1,7 +1,6 @@
import pprint
import six
import signal
import logging
import pprint
import signal
import warnings
from twisted.internet import reactor, defer
@ -22,6 +21,7 @@ from scrapy.utils.log import (
get_scrapy_root_handler, install_scrapy_root_handler)
from scrapy import signals
logger = logging.getLogger(__name__)
@ -206,7 +206,7 @@ class CrawlerRunner(object):
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
if isinstance(spidercls, six.string_types):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)

4
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import re
import logging
import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@ -67,7 +65,7 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
def _has_ajaxcrawlable_meta(text):

9
scrapy/downloadermiddlewares/cookies.py
View File

@ -1,5 +1,3 @@
import os
import six
import logging
from collections import defaultdict
@ -8,6 +6,7 @@ from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@ -82,8 +81,10 @@ class CookiesMiddleware(object):
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
cookie_list = [{'name': k, 'value': v} for k, v in \
six.iteritems(request.cookies)]
cookie_list = [
{'name': k, 'value': v}
for k, v in request.cookies.items()
]
else:
cookie_list = request.cookies

11
scrapy/downloadermiddlewares/decompression.py
View File

@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
from io import BytesIO
import zipfile
import tarfile
import logging
import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
import six
from scrapy.responsetypes import responsetypes
logger = logging.getLogger(__name__)
@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
if not response.body:
return response
for fmt, func in six.iteritems(self._formats):
for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',

16
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,11 +1,19 @@
from email.utils import formatdate
from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
ConnectionRefusedError, ConnectionDone, ConnectError, \
ConnectionLost, TCPTimedOutError
from twisted.internet.error import (
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
DNSLookupError,
TCPTimedOutError,
TimeoutError,
)
from twisted.web.client import ResponseFailed
from scrapy import signals
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.utils.misc import load_object

5
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
respcls = responsetypes.from_args(headers=response.headers, \
url=response.url, body=decoded_body)
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the

5
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -1,7 +1,6 @@
import base64
from six.moves.urllib.parse import unquote, urlunparse
from six.moves.urllib.request import getproxies, proxy_bypass
from urllib.request import _parse_proxy
from urllib.parse import unquote, urlunparse
from urllib.request import getproxies, proxy_bypass, _parse_proxy
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached

3
scrapy/downloadermiddlewares/redirect.py
View File

@ -1,5 +1,5 @@
import logging
from six.moves.urllib.parse import urljoin, urlparse
from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)

1
scrapy/dupefilters.py
View File

@ -1,4 +1,3 @@
from __future__ import print_function
import os
import logging

17
scrapy/exporters.py
View File

@ -6,15 +6,14 @@ import csv
import io
import pprint
import marshal
import six
from six.moves import cPickle as pickle
import warnings
import pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@ -60,9 +59,9 @@ class BaseItemExporter(object):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
field_iter = six.iterkeys(item.fields)
field_iter = item.fields.keys()
else:
field_iter = six.iterkeys(item)
field_iter = item.keys()
else:
if include_empty:
field_iter = self.fields_to_export
@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter):
for value in serialized_value:
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
elif isinstance(serialized_value, str):
self.xg.characters(serialized_value)
else:
self.xg.characters(str(serialized_value))
@ -200,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
)
self.csv_writer = csv.writer(self.stream, **kwargs)
self._headers_not_written = True
@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (six.text_type, bytes)):
if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
def _serialize_dict(self, value):
for key, val in six.iteritems(value):
for key, val in value.items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)

6
scrapy/extensions/closespider.py
View File

@ -54,9 +54,9 @@ class CloseSpider(object):
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
def spider_opened(self, spider):
self.task = reactor.callLater(self.close_on['timeout'], \
self.crawler.engine.close_spider, spider, \
reason='closespider_timeout')
self.task = reactor.callLater(self.close_on['timeout'],
self.crawler.engine.close_spider, spider,
reason='closespider_timeout')
def item_scraped(self, item, spider):
self.counter['itemcount'] += 1

2
scrapy/extensions/feedexport.py
View File

@ -10,7 +10,7 @@ import logging
import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
from six.moves.urllib.parse import urlparse, unquote
from urllib.parse import urlparse, unquote
from ftplib import FTP
from zope.interface import Interface, implementer

4
scrapy/extensions/httpcache.py
View File

@ -1,14 +1,12 @@
from __future__ import print_function
import gzip
import logging
import os
import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from time import time
from weakref import WeakKeyDictionary
from six.moves import cPickle as pickle
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response

3
scrapy/extensions/memdebug.py
View File

@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
"""
import gc
import six
from scrapy import signals
from scrapy.exceptions import NotConfigured
@ -28,7 +27,7 @@ class MemoryDebugger(object):
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
for cls, wdict in six.iteritems(live_refs):
for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)

2
scrapy/extensions/spiderstate.py
View File

@ -1,5 +1,5 @@
import os
from six.moves import cPickle as pickle
import pickle
from scrapy import signals
from scrapy.exceptions import NotConfigured

5
scrapy/http/cookies.py
View File

@ -1,7 +1,6 @@
import time
from six.moves.http_cookiejar import (
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
)
from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode

10
scrapy/http/headers.py
View File

@ -1,4 +1,3 @@
import six
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@ -19,7 +18,7 @@ class Headers(CaselessDict):
"""Normalize values to bytes"""
if value is None:
value = []
elif isinstance(value, (six.text_type, bytes)):
elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
@ -29,10 +28,10 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
elif isinstance(x, six.text_type):
elif isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
return six.text_type(x).encode(self.encoding)
return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
@ -68,9 +67,6 @@ class Headers(CaselessDict):
self[key] = lst
def items(self):
return list(self.iteritems())
def iteritems(self):
return ((k, self.getlist(k)) for k in self.keys())
def values(self):

3
scrapy/http/request/__init__.py
View File

@ -4,7 +4,6 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
import six
from w3lib.url import safe_url_string
from scrapy.http.headers import Headers
@ -60,7 +59,7 @@ class Request(object_ref):
return self._url
def _set_url(self, url):
if not isinstance(url, six.string_types):
if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
s = safe_url_string(url, self.encoding)

5
scrapy/http/request/form.py
View File

@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
import six
from six.moves.urllib.parse import urljoin, urlencode
from urllib.parse import urljoin, urlencode
import lxml.html
from parsel.selector import create_root_node
@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + \
u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')

2
scrapy/http/request/rpc.py
View File

@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
from six.moves import xmlrpc_client as xmlrpclib
import xmlrpc.client as xmlrpclib
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args

6
scrapy/http/response/__init__.py
View File

@ -4,7 +4,7 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
from scrapy.http.request import Request
from scrapy.http.headers import Headers
@ -113,8 +113,8 @@ class Response(object_ref):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""

13
scrapy/http/response/text.py
View File

@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
import six
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
@ -31,14 +30,14 @@ class TextResponse(Response):
super(TextResponse, self).__init__(*args, **kwargs)
def _set_url(self, url):
if isinstance(url, six.text_type):
if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
if isinstance(body, six.text_type):
if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
@ -126,7 +125,7 @@ class TextResponse(Response):
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
@ -134,7 +133,7 @@ class TextResponse(Response):
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` or ``<link>`` element, e.g.
``response.css('a.my_link')[0]``.
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
@ -158,7 +157,7 @@ class TextResponse(Response):
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
if isinstance(sel.root, six.string_types):
if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):

7
scrapy/item.py
View File

@ -10,8 +10,6 @@ from copy import deepcopy
from pprint import pformat
from warnings import warn
import six
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
@ -78,7 +76,7 @@ class DictItem(MutableMapping, BaseItem):
def __init__(self, *args, **kwargs):
self._values = {}
if args or kwargs: # avoid creating dict for most common case
for k, v in six.iteritems(dict(*args, **kwargs)):
for k, v in dict(*args, **kwargs).items():
self[k] = v
def __getitem__(self, key):
@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
return deepcopy(self)
@six.add_metaclass(ItemMeta)
class Item(DictItem):
class Item(DictItem, metaclass=ItemMeta):
pass

18
scrapy/linkextractors/__init__.py
View File

@ -6,11 +6,13 @@ This package contains a collection of Link Extractors.
For more info see docs/topics/link-extractors.rst
"""
import re
from urllib.parse import urlparse
from warnings import warn
from six.moves.urllib.parse import urlparse
from parsel.csstranslator import HTMLTranslator
from w3lib.url import canonicalize_url
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import (
url_is_from_any_domain, url_has_any_extension,
@ -44,14 +46,22 @@ IGNORED_EXTENSIONS = [
_re_type = type(re.compile("", 0))
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \
'file', 'ftp'}
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor(object):
_csstranslator = HTMLTranslator()
def __new__(cls, *args, **kwargs):
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
if (issubclass(cls, FilteringLinkExtractor) and
not issubclass(cls, LxmlLinkExtractor)):
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(FilteringLinkExtractor, cls).__new__(cls)
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
@ -118,4 +128,4 @@ class FilteringLinkExtractor(object):
# Top-level imports
from .lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401

7
scrapy/linkextractors/htmlparser.py
View File

@ -2,9 +2,8 @@
HTMLParser-based link extractor
"""
import warnings
import six
from six.moves.html_parser import HTMLParser
from six.moves.urllib.parse import urljoin
from html.parser import HTMLParser
from urllib.parse import urljoin
from w3lib.url import safe_url_string
from w3lib.html import strip_html5_whitespace
@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser):
ret = []
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in links:
if isinstance(link.url, six.text_type):
if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)

13
scrapy/linkextractors/lxmlhtml.py
View File

@ -1,8 +1,7 @@
"""
Link extractor based on lxml.html
"""
import six
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()")
def _nons(tag):
if isinstance(tag, six.string_types):
if isinstance(tag, str):
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
return tag.split('}')[-1]
return tag
@ -117,6 +116,14 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
restrict_text=restrict_text)
def extract_links(self, response):
"""Returns a list of :class:`~scrapy.link.Link` objects from the
specified :class:`response <scrapy.http.Response>`.
Only links that match the settings passed to the ``__init__`` method of
the link extractor are returned.
Duplicate links are omitted.
"""
base_url = get_base_url(response)
if self.restrict_xpaths:
docs = [subdoc

5
scrapy/linkextractors/regex.py
View File

@ -1,10 +1,11 @@
import re
from six.moves.urllib.parse import urljoin
from urllib.parse import urljoin
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
from scrapy.link import Link
from .sgml import SgmlLinkExtractor
from scrapy.linkextractors.sgml import SgmlLinkExtractor
linkre = re.compile(
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",

5
scrapy/linkextractors/sgml.py
View File

@ -1,9 +1,8 @@
"""
SGMLParser-based Link extractors
"""
import six
from six.moves.urllib.parse import urljoin
import warnings
from urllib.parse import urljoin
from sgmllib import SGMLParser
from w3lib.url import safe_url_string, canonicalize_url
@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
if isinstance(link.url, six.text_type):
if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)

4
scrapy/logformatter.py
View File

@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'"
class LogFormatter(object):
"""Class for generating log messages for different actions.
All methods must return a dictionary listing the parameters ``level``, ``msg``
and ``args`` which are going to be used for constructing the log message when
calling ``logging.log``.
@ -48,7 +48,7 @@ class LogFormatter(object):
}
}
"""
def crawled(self, request, response, spider):
"""Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else ''

17
scrapy/mail.py
View File

@ -3,21 +3,21 @@ Mail sending helpers
See documentation in docs/topics/email.rst
"""
from io import BytesIO
import logging
from email.utils import COMMASPACE, formatdate
from six.moves.email_mime_multipart import MIMEMultipart
from six.moves.email_mime_text import MIMEText
from six.moves.email_mime_base import MIMEBase
from email.mime.nonmultipart import MIMENonMultipart
from email import encoders as Encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.nonmultipart import MIMENonMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
from io import BytesIO
from twisted.internet import defer, reactor, ssl
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
logger = logging.getLogger(__name__)
@ -73,8 +73,7 @@ class MailSender(object):
part = MIMEBase(*mimetype.split('/'))
part.set_payload(f.read())
Encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="%s"' \
% attach_name)
part.add_header('Content-Disposition', 'attachment', filename=attach_name)
msg.attach(part)
else:
msg.set_payload(body)

4
scrapy/middleware.py
View File

@ -65,8 +65,8 @@ class MiddlewareManager(object):
return process_chain(self.methods[methodname], obj, *args)
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
return process_chain_both(self.methods[cb_methodname], \
self.methods[eb_methodname], obj, *args)
return process_chain_both(self.methods[cb_methodname],
self.methods[eb_methodname], obj, *args)
def open_spider(self, spider):
return self._process_parallel('open_spider', spider)

19
scrapy/pipelines/files.py
View File

@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
from io import BytesIO
import logging
import mimetypes
import os
import os.path
import time
import logging
from email.utils import parsedate_tz, mktime_tz
from six.moves.urllib.parse import urlparse
from collections import defaultdict
import six
from email.utils import parsedate_tz, mktime_tz
from io import BytesIO
from urllib.parse import urlparse
from twisted.internet import defer, threads
@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
logger = logging.getLogger(__name__)
@ -153,14 +152,14 @@ class S3FilesStore(object):
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
for metakey, metavalue in six.iteritems(meta):
for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
@ -201,7 +200,7 @@ class S3FilesStore(object):
'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
})
extra = {}
for key, value in six.iteritems(headers):
for key, value in headers.items():
try:
kwarg = mapping[key]
except KeyError:
@ -249,7 +248,7 @@ class GCSFilesStore(object):
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),

3
scrapy/pipelines/images.py
View File

@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
import functools
import hashlib
from io import BytesIO
import six
from PIL import Image
@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
image, buf = self.convert_image(orig_image)
yield path, image, buf
for thumb_id, size in six.iteritems(self.thumbs):
for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf

2
scrapy/pipelines/media.py
View File

@ -1,5 +1,3 @@
from __future__ import print_function
import functools
import logging
from collections import defaultdict

4
scrapy/responsetypes.py
View File

@ -2,11 +2,9 @@
This module implements a class which returns the appropriate Response class
based on different criteria.
"""
from __future__ import absolute_import
from mimetypes import MimeTypes
from pkgutil import get_data
from io import StringIO
import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
@ -37,7 +35,7 @@ class ResponseTypes(object):
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in six.iteritems(self.CLASSES):
for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):

5
scrapy/robotstxt.py
View File

@ -1,7 +1,6 @@
import sys
import logging
from abc import ABCMeta, abstractmethod
from six import with_metaclass
from scrapy.utils.python import to_unicode
@ -26,7 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
return robotstxt_body
class RobotParser(with_metaclass(ABCMeta)):
class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):
@ -56,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
from six.moves.urllib_robotparser import RobotFileParser
from urllib.robotparser import RobotFileParser
self.spider = spider
robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp = RobotFileParser()

19
scrapy/settings/__init__.py
View File

@ -1,4 +1,3 @@
import six
import json
import copy
from collections.abc import MutableMapping
@ -23,7 +22,7 @@ def get_settings_priority(priority):
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
numerical value, or directly returns a given numerical priority.
"""
if isinstance(priority, six.string_types):
if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
@ -173,7 +172,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or [])
if isinstance(value, six.string_types):
if isinstance(value, str):
value = value.split(',')
return list(value)
@ -194,7 +193,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or {})
if isinstance(value, six.string_types):
if isinstance(value, str):
value = json.loads(value)
return dict(value)
@ -284,7 +283,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
if isinstance(module, six.string_types):
if isinstance(module, str):
module = import_module(module)
for key in dir(module):
if key.isupper():
@ -313,14 +312,14 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
if isinstance(values, six.string_types):
if isinstance(values, str):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
for name, value in six.iteritems(values):
for name, value in values.items():
self.set(name, value, values.getpriority(name))
else:
for name, value in six.iteritems(values):
for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority='project'):
@ -377,7 +376,7 @@ class BaseSettings(MutableMapping):
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in six.iteritems(self)}
for k, v in self.items()}
def copy_to_dict(self):
"""
@ -445,7 +444,7 @@ class Settings(BaseSettings):
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities
for name, val in six.iteritems(self):
for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)

4
scrapy/settings/default_settings.py
View File

@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
# also allowing negotiation
# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
DOWNLOADER_MIDDLEWARES = {}

2
scrapy/shell.py
View File

@ -3,8 +3,6 @@
See documentation in docs/topics/shell.rst
"""
from __future__ import print_function
import os
import signal
import warnings

1
scrapy/signalmanager.py
View File

@ -1,4 +1,3 @@
from __future__ import absolute_import
from pydispatch import dispatcher
from scrapy.utils import signal as _signal

1
scrapy/spiderloader.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from collections import defaultdict
import traceback
import warnings

Some files were not shown because too many files have changed in this diff Show More