mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into documentation-coverage
This commit is contained in:
Adrián Chaves
GitHub
commit
1fc2b140c1
|
|
@ -0,0 +1,18 @@
|
|||
skips:
|
||||
- B101
|
||||
- B105
|
||||
- B301
|
||||
- B303
|
||||
- B306
|
||||
- B307
|
||||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B402
|
||||
- B403
|
||||
- B404
|
||||
- B406
|
||||
- B410
|
||||
- B503
|
||||
- B603
|
||||
- B605
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 1.7.0
|
||||
current_version = 1.8.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ about: Report a problem to help us improve
|
|||
Thanks for taking an interest in Scrapy!
|
||||
|
||||
If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
|
||||
The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
|
||||
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
|
||||
|
||||
Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ about: Suggest an idea for an enhancement or new feature
|
|||
Thanks for taking an interest in Scrapy!
|
||||
|
||||
If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
|
||||
The Github issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
|
||||
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
|
||||
|
||||
Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
|
||||
|
||||
|
|
|
|||
27
.travis.yml
27
.travis.yml
|
|
@ -7,14 +7,10 @@ branches:
|
|||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=py27
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-pinned
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-extra-deps
|
||||
python: 2.7
|
||||
- env: TOXENV=pypy
|
||||
python: 2.7
|
||||
- env: TOXENV=security
|
||||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pypy3
|
||||
python: 3.5
|
||||
- env: TOXENV=py35
|
||||
|
|
@ -25,19 +21,14 @@ matrix:
|
|||
python: 3.6
|
||||
- env: TOXENV=py37
|
||||
python: 3.7
|
||||
- env: TOXENV=py37-extra-deps
|
||||
python: 3.7
|
||||
- env: TOXENV=py38
|
||||
python: 3.8
|
||||
- env: TOXENV=py38-extra-deps
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.6
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy" ]; then
|
||||
export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
|
|
@ -68,4 +59,4 @@ deploy:
|
|||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ members of the project's leadership.
|
|||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at [http://contributor-covenant.org/version/1/4][version]
|
||||
available at [http://contributor-covenant.org/version/1/4][version].
|
||||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
|
|
|
|||
20
README.rst
20
README.rst
|
|
@ -34,13 +34,13 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
|
|||
crawl websites and extract structured data from their pages. It can be used for
|
||||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
For more information including a list of features check the Scrapy homepage at:
|
||||
https://scrapy.org
|
||||
Check the Scrapy homepage at https://scrapy.org for more information,
|
||||
including a list of features.
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* Python 2.7 or Python 3.5+
|
||||
* Python 3.5+
|
||||
* Works on Linux, Windows, Mac OSX, BSD
|
||||
|
||||
Install
|
||||
|
|
@ -50,8 +50,8 @@ The quick way::
|
|||
|
||||
pip install scrapy
|
||||
|
||||
For more details see the install section in the documentation:
|
||||
https://docs.scrapy.org/en/latest/intro/install.html
|
||||
See the install section in the documentation at
|
||||
https://docs.scrapy.org/en/latest/intro/install.html for more details.
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
|
@ -62,17 +62,17 @@ directory.
|
|||
Releases
|
||||
========
|
||||
|
||||
You can find release notes at https://docs.scrapy.org/en/latest/news.html
|
||||
You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
|
||||
|
||||
Community (blog, twitter, mail list, IRC)
|
||||
=========================================
|
||||
|
||||
See https://scrapy.org/community/
|
||||
See https://scrapy.org/community/ for details.
|
||||
|
||||
Contributing
|
||||
============
|
||||
|
||||
See https://docs.scrapy.org/en/master/contributing.html
|
||||
See https://docs.scrapy.org/en/master/contributing.html for details.
|
||||
|
||||
Code of Conduct
|
||||
---------------
|
||||
|
|
@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
|
|||
Companies using Scrapy
|
||||
======================
|
||||
|
||||
See https://scrapy.org/companies/
|
||||
See https://scrapy.org/companies/ for a list.
|
||||
|
||||
Commercial Support
|
||||
==================
|
||||
|
||||
See https://scrapy.org/support/
|
||||
See https://scrapy.org/support/ for details.
|
||||
|
|
|
|||
20
conftest.py
20
conftest.py
|
|
@ -1,4 +1,3 @@
|
|||
import six
|
||||
import pytest
|
||||
|
||||
|
||||
|
|
@ -8,14 +7,23 @@ collect_ignore = [
|
|||
]
|
||||
|
||||
|
||||
if six.PY3:
|
||||
for line in open('tests/py3-ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
for line in open('tests/ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def chdir(tmpdir):
|
||||
"""Change to pytest-provided temporary directory"""
|
||||
tmpdir.chdir()
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(session, config, items):
|
||||
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
|
||||
try:
|
||||
from pytest_flake8 import Flake8Item
|
||||
if config.getoption('--flake8'):
|
||||
items[:] = [item for item in items if isinstance(item, Flake8Item)]
|
||||
except ImportError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1,281 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Quotes to Scrape</title>
|
||||
<link rel="stylesheet" href="/static/bootstrap.min.css">
|
||||
<link rel="stylesheet" href="/static/main.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="row header-box">
|
||||
<div class="col-md-8">
|
||||
<h1>
|
||||
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
|
||||
</h1>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<p>
|
||||
|
||||
<a href="/login">Login</a>
|
||||
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-8">
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
|
||||
|
||||
<a class="tag" href="/tag/change/page/1/">change</a>
|
||||
|
||||
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
|
||||
|
||||
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
|
||||
|
||||
<a class="tag" href="/tag/world/page/1/">world</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
|
||||
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
|
||||
<a href="/author/J-K-Rowling">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
|
||||
|
||||
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
|
||||
|
||||
<a class="tag" href="/tag/choices/page/1/">choices</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||
|
||||
<a class="tag" href="/tag/live/page/1/">live</a>
|
||||
|
||||
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
|
||||
|
||||
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
|
||||
<span>by <small class="author" itemprop="author">Jane Austen</small>
|
||||
<a href="/author/Jane-Austen">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
|
||||
|
||||
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
|
||||
|
||||
<a class="tag" href="/tag/books/page/1/">books</a>
|
||||
|
||||
<a class="tag" href="/tag/classic/page/1/">classic</a>
|
||||
|
||||
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</span>
|
||||
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
|
||||
<a href="/author/Marilyn-Monroe">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
|
||||
|
||||
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
|
||||
|
||||
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
|
||||
|
||||
<a class="tag" href="/tag/success/page/1/">success</a>
|
||||
|
||||
<a class="tag" href="/tag/value/page/1/">value</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
|
||||
<span>by <small class="author" itemprop="author">André Gide</small>
|
||||
<a href="/author/Andre-Gide">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="life,love" / >
|
||||
|
||||
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||
|
||||
<a class="tag" href="/tag/love/page/1/">love</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“I have not failed. I've just found 10,000 ways that won't work.”</span>
|
||||
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
|
||||
<a href="/author/Thomas-A-Edison">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
|
||||
|
||||
<a class="tag" href="/tag/edison/page/1/">edison</a>
|
||||
|
||||
<a class="tag" href="/tag/failure/page/1/">failure</a>
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</span>
|
||||
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
|
||||
<a href="/author/Eleanor-Roosevelt">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
|
||||
|
||||
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
|
||||
<span>by <small class="author" itemprop="author">Steve Martin</small>
|
||||
<a href="/author/Steve-Martin">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
|
||||
|
||||
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||
|
||||
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
|
||||
|
||||
<a class="tag" href="/tag/simile/page/1/">simile</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav>
|
||||
<ul class="pager">
|
||||
|
||||
|
||||
<li class="next">
|
||||
<a href="/page/2/">Next <span aria-hidden="true">→</span></a>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="col-md-4 tags-box">
|
||||
|
||||
<h2>Top Ten tags</h2>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
|
||||
</span>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<footer class="footer">
|
||||
<div class="container">
|
||||
<p class="text-muted">
|
||||
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
|
||||
</p>
|
||||
<p class="copyright">
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
|
||||
</p>
|
||||
</div>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
16
docs/conf.py
16
docs/conf.py
|
|
@ -27,10 +27,13 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
|
|||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = [
|
||||
'hoverxref.extension',
|
||||
'notfound.extension',
|
||||
'scrapydocs',
|
||||
'sphinx.ext.autodoc',
|
||||
'sphinx.ext.coverage',
|
||||
'sphinx.ext.intersphinx',
|
||||
'sphinx.ext.viewcode',
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
|
|
@ -237,7 +240,7 @@ coverage_ignore_pyobjects = [
|
|||
r'\bContractsManager\b$',
|
||||
|
||||
# For default contracts we only want to document their general purpose in
|
||||
# their constructor, the methods they reimplement to achieve that purpose
|
||||
# their __init__ method, the methods they reimplement to achieve that purpose
|
||||
# should be irrelevant to developers using those contracts.
|
||||
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
|
||||
|
||||
|
|
@ -276,5 +279,16 @@ coverage_ignore_pyobjects = [
|
|||
# -------------------------------------
|
||||
|
||||
intersphinx_mapping = {
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
|
||||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
}
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
# ------------------------------------
|
||||
|
||||
hoverxref_auto_ref = True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
import os
|
||||
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
||||
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from sybil import Sybil
|
||||
from sybil.parsers.codeblock import CodeBlockParser
|
||||
from sybil.parsers.doctest import DocTestParser
|
||||
from sybil.parsers.skip import skip
|
||||
|
||||
|
||||
def load_response(url, filename):
|
||||
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
|
||||
with open(input_path, 'rb') as input_file:
|
||||
return HtmlResponse(url, body=input_file.read())
|
||||
|
||||
|
||||
def setup(namespace):
|
||||
namespace['load_response'] = load_response
|
||||
|
||||
|
||||
pytest_collect_file = Sybil(
|
||||
parsers=[
|
||||
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||
CodeBlockParser(future_imports=['print_function']),
|
||||
skip,
|
||||
],
|
||||
pattern='*.rst',
|
||||
setup=setup,
|
||||
).pytest()
|
||||
|
|
@ -177,33 +177,33 @@ Documentation policies
|
|||
======================
|
||||
|
||||
For reference documentation of API members (classes, methods, etc.) use
|
||||
docstrings and make sure that the Sphinx documentation uses the autodoc_
|
||||
extension to pull the docstrings. API reference documentation should follow
|
||||
docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
|
||||
and it may provide short examples.
|
||||
docstrings and make sure that the Sphinx documentation uses the
|
||||
:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
|
||||
documentation should follow docstring conventions (`PEP 257`_) and be
|
||||
IDE-friendly: short, to the point, and it may provide short examples.
|
||||
|
||||
Other types of documentation, such as tutorials or topics, should be covered in
|
||||
files within the ``docs/`` directory. This includes documentation that is
|
||||
specific to an API member, but goes beyond API reference documentation.
|
||||
|
||||
In any case, if something is covered in a docstring, use the autodoc_
|
||||
extension to pull the docstring into the documentation instead of duplicating
|
||||
the docstring in files within the ``docs/`` directory.
|
||||
|
||||
.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
|
||||
In any case, if something is covered in a docstring, use the
|
||||
:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
|
||||
documentation instead of duplicating the docstring in files within the
|
||||
``docs/`` directory.
|
||||
|
||||
Tests
|
||||
=====
|
||||
|
||||
Tests are implemented using the `Twisted unit-testing framework`_, running
|
||||
tests requires `tox`_.
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:core/development/policy/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
|
||||
.. _running-tests:
|
||||
|
||||
Running tests
|
||||
-------------
|
||||
|
||||
Make sure you have a recent enough `tox`_ installation:
|
||||
Make sure you have a recent enough :doc:`tox <tox:index>` installation:
|
||||
|
||||
``tox --version``
|
||||
|
||||
|
|
@ -219,37 +219,37 @@ To run a specific test (say ``tests/test_loader.py``) use:
|
|||
|
||||
``tox -- tests/test_loader.py``
|
||||
|
||||
To run the tests on a specific tox_ environment, use ``-e <name>`` with an
|
||||
environment name from ``tox.ini``. For example, to run the tests with Python
|
||||
3.6 use::
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
|
||||
tox -e py36
|
||||
|
||||
You can also specify a comma-separated list of environmets, and use `tox’s
|
||||
parallel mode`_ to run the tests on multiple environments in parallel::
|
||||
You can also specify a comma-separated list of environmets, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py27,py36 -p auto
|
||||
|
||||
To pass command-line options to pytest_, add them after ``--`` in your call to
|
||||
tox_. Using ``--`` overrides the default positional arguments defined in
|
||||
``tox.ini``, so you must include those default positional arguments
|
||||
(``scrapy tests``) after ``--`` as well::
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
default positional arguments defined in ``tox.ini``, so you must include those
|
||||
default positional arguments (``scrapy tests``) after ``--`` as well::
|
||||
|
||||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 tox_ environment using all your CPU cores::
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install `coverage`_ (``pip install coverage``) and run:
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
||||
``coverage report``
|
||||
|
||||
see output of ``coverage --help`` for more options like html or xml report.
|
||||
|
||||
.. _coverage: https://pypi.python.org/pypi/coverage
|
||||
|
||||
Writing tests
|
||||
-------------
|
||||
|
||||
|
|
@ -270,13 +270,9 @@ And their unit-tests are in::
|
|||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://reddit.com/r/scrapy
|
||||
.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
|
||||
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
|
||||
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
|
||||
.. _open issues: https://github.com/scrapy/scrapy/issues
|
||||
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
|
||||
.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
|
||||
.. _pytest: https://docs.pytest.org/en/latest/usage.html
|
||||
.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
|
||||
.. _tox: https://pypi.python.org/pypi/tox
|
||||
.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
|
||||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||
|
|
|
|||
|
|
@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 2.7 and Python 3.5+
|
||||
Scrapy is supported under Python 3.5+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 2.6 support was dropped starting at Scrapy 0.20.
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
Python 2 support was dropped in Scrapy 2.0.
|
||||
|
||||
.. note::
|
||||
For Python 3 support on Windows, it is recommended to use
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Installation guide
|
|||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 2.7 and Python 3.5 or above
|
||||
Scrapy runs on Python 3.5 or above
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
|
|
@ -102,10 +102,8 @@ just like any other Python package.
|
|||
(See :ref:`platform-specific guides <intro-install-platform-notes>`
|
||||
below for non-Python dependencies that you may need to install beforehand).
|
||||
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
|
||||
|
||||
* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
|
||||
* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
|
||||
only supports Python 3, make sure you created a Python 3 virtualenv.
|
||||
|
||||
.. _virtualenv: https://virtualenv.pypa.io
|
||||
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
|
||||
|
|
@ -149,16 +147,12 @@ typically too old and slow to catch up with latest Scrapy.
|
|||
To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
|
||||
these dependencies::
|
||||
|
||||
sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
|
||||
- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
|
||||
- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
|
||||
are required for ``lxml``
|
||||
- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
|
||||
|
||||
If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
|
||||
|
||||
sudo apt-get install python3 python3-dev
|
||||
|
||||
Inside a :ref:`virtualenv <intro-using-virtualenv>`,
|
||||
you can install Scrapy with ``pip`` after that::
|
||||
|
||||
|
|
@ -290,5 +284,5 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
|
|||
.. _zsh: https://www.zsh.org/
|
||||
.. _Scrapinghub: https://scrapinghub.com
|
||||
.. _Anaconda: https://docs.anaconda.com/anaconda/
|
||||
.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
|
||||
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.xpath('span/small/text()').get(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr("href")').get()
|
||||
|
|
|
|||
|
|
@ -78,9 +78,9 @@ Our first Spider
|
|||
|
||||
Spiders are classes that you define and that Scrapy uses to scrape information
|
||||
from a website (or a group of websites). They must subclass
|
||||
:class:`scrapy.Spider` and define the initial requests to make, optionally how
|
||||
to follow links in the pages, and how to parse the downloaded page content to
|
||||
extract data.
|
||||
:class:`~scrapy.spiders.Spider` and define the initial requests to make,
|
||||
optionally how to follow links in the pages, and how to parse the downloaded
|
||||
page content to extract data.
|
||||
|
||||
This is the code for our first Spider. Save it in a file named
|
||||
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
|
||||
|
|
@ -235,13 +235,16 @@ You will see something like::
|
|||
[s] shelp() Shell help (print this help)
|
||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||
[s] view(response) View response in a browser
|
||||
>>>
|
||||
|
||||
Using the shell, you can try selecting elements using `CSS`_ with the response
|
||||
object::
|
||||
object:
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
||||
The result of running ``response.css('title')`` is a list-like object called
|
||||
:class:`~scrapy.selector.SelectorList`, which represents a list of
|
||||
|
|
@ -372,6 +375,9 @@ we want::
|
|||
We get a list of selectors for the quote HTML elements with::
|
||||
|
||||
>>> response.css("div.quote")
|
||||
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||
...]
|
||||
|
||||
Each of the selectors returned by the query above allows us to run further
|
||||
queries over their sub-elements. Let's assign the first selector to a
|
||||
|
|
@ -396,6 +402,12 @@ to get all of them::
|
|||
>>> tags
|
||||
['change', 'deep-thoughts', 'thinking', 'world']
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from sys import version_info
|
||||
|
||||
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
|
||||
|
||||
Having figured out how to extract each bit, we can now iterate over all the
|
||||
quotes elements and put them together into a Python dictionary::
|
||||
|
||||
|
|
@ -404,10 +416,9 @@ quotes elements and put them together into a Python dictionary::
|
|||
... author = quote.css("small.author::text").get()
|
||||
... tags = quote.css("div.tags a.tag::text").getall()
|
||||
... print(dict(text=text, author=author, tags=tags))
|
||||
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
|
||||
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
|
||||
... a few more of these, omitted for brevity
|
||||
>>>
|
||||
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
|
||||
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
|
||||
...
|
||||
|
||||
Extracting data in our spider
|
||||
-----------------------------
|
||||
|
|
@ -521,7 +532,7 @@ There is also an ``attrib`` property available
|
|||
(see :ref:`selecting-attributes` for more)::
|
||||
|
||||
>>> response.css('li.next a').attrib['href']
|
||||
'/page/2'
|
||||
'/page/2/'
|
||||
|
||||
Let's see now our spider modified to recursively follow the link to the next
|
||||
page, extracting data from it::
|
||||
|
|
|
|||
294
docs/news.rst
294
docs/news.rst
|
|
@ -6,22 +6,246 @@ Release notes
|
|||
.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
|
||||
planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
|
||||
|
||||
.. _release-1.8.0:
|
||||
|
||||
Scrapy 1.8.0 (2019-10-28)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* Dropped Python 3.4 support and updated minimum requirements; made Python 3.8
|
||||
support official
|
||||
* New :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class method
|
||||
* New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings
|
||||
* New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
|
||||
:setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Python 3.4 is no longer supported, and some of the minimum requirements of
|
||||
Scrapy have also changed:
|
||||
|
||||
* cssselect_ 0.9.1
|
||||
* cryptography_ 2.0
|
||||
* lxml_ 3.5.0
|
||||
* pyOpenSSL_ 16.2.0
|
||||
* queuelib_ 1.4.2
|
||||
* service_identity_ 16.0.0
|
||||
* six_ 1.10.0
|
||||
* Twisted_ 17.9.0 (16.0.0 with Python 2)
|
||||
* zope.interface_ 4.1.3
|
||||
|
||||
(:issue:`3892`)
|
||||
|
||||
* ``JSONRequest`` is now called :class:`~scrapy.http.JsonRequest` for
|
||||
consistency with similar classes (:issue:`3929`, :issue:`3982`)
|
||||
|
||||
* If you are using a custom context factory
|
||||
(:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
|
||||
accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
|
||||
(:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
|
||||
|
||||
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
|
||||
into lists::
|
||||
|
||||
>>> item = MyItem()
|
||||
>>> item['field'] = 'value1'
|
||||
>>> loader = ItemLoader(item=item)
|
||||
>>> item['field']
|
||||
['value1']
|
||||
|
||||
This is needed to allow adding values to existing fields
|
||||
(``loader.add_value('field', 'value2')``).
|
||||
|
||||
(:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
|
||||
:issue:`4036`)
|
||||
|
||||
See also :ref:`1.8-deprecation-removals` below.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* A new :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class
|
||||
method allows :ref:`creating a request from a cURL command
|
||||
<requests-from-curl>` (:issue:`2985`, :issue:`3862`)
|
||||
|
||||
* A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
|
||||
parser to use. It includes built-in support for
|
||||
:ref:`RobotFileParser <python-robotfileparser>`,
|
||||
:ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and
|
||||
:ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to
|
||||
:ref:`implement support for additional parsers
|
||||
<support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`,
|
||||
:issue:`3796`, :issue:`3935`, :issue:`3969`, :issue:`4006`)
|
||||
|
||||
* A new :setting:`ROBOTSTXT_USER_AGENT` setting allows defining a separate
|
||||
user agent string to use for robots.txt_ parsing (:issue:`3931`,
|
||||
:issue:`3966`)
|
||||
|
||||
* :class:`~scrapy.spiders.Rule` no longer requires a :class:`LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` parameter
|
||||
(:issue:`781`, :issue:`4016`)
|
||||
|
||||
* Use the new :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting to customize
|
||||
the TLS/SSL ciphers used by the default HTTP/1.1 downloader (:issue:`3392`,
|
||||
:issue:`3442`)
|
||||
|
||||
* Set the new :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` setting to
|
||||
``True`` to enable debug-level messages about TLS connection parameters
|
||||
after establishing HTTPS connections (:issue:`2111`, :issue:`3450`)
|
||||
|
||||
* Callbacks that receive keyword arguments
|
||||
(see :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>`) can now be
|
||||
tested using the new :class:`@cb_kwargs
|
||||
<scrapy.contracts.default.CallbackKeywordArgumentsContract>`
|
||||
:ref:`spider contract <topics-contracts>` (:issue:`3985`, :issue:`3988`)
|
||||
|
||||
* When a :class:`@scrapes <scrapy.contracts.default.ScrapesContract>` spider
|
||||
contract fails, all missing fields are now reported (:issue:`766`,
|
||||
:issue:`3939`)
|
||||
|
||||
* :ref:`Custom log formats <custom-log-formats>` can now drop messages by
|
||||
having the corresponding methods of the configured :setting:`LOG_FORMATTER`
|
||||
return ``None`` (:issue:`3984`, :issue:`3987`)
|
||||
|
||||
* A much improved completion definition is now available for Zsh_
|
||||
(:issue:`4069`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` no
|
||||
longer makes later calls to :meth:`ItemLoader.get_output_value()
|
||||
<scrapy.loader.ItemLoader.get_output_value>` or
|
||||
:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` return
|
||||
empty data (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`,
|
||||
:issue:`3998`, :issue:`4036`)
|
||||
|
||||
* Fixed :class:`~scrapy.statscollectors.DummyStatsCollector` raising a
|
||||
:exc:`TypeError` exception (:issue:`4007`, :issue:`4052`)
|
||||
|
||||
* :meth:`FilesPipeline.file_path
|
||||
<scrapy.pipelines.files.FilesPipeline.file_path>` and
|
||||
:meth:`ImagesPipeline.file_path
|
||||
<scrapy.pipelines.images.ImagesPipeline.file_path>` no longer choose
|
||||
file extensions that are not `registered with IANA`_ (:issue:`1287`,
|
||||
:issue:`3953`, :issue:`3954`)
|
||||
|
||||
* When using botocore_ to persist files in S3, all botocore-supported headers
|
||||
are properly mapped now (:issue:`3904`, :issue:`3905`)
|
||||
|
||||
* FTP passwords in :setting:`FEED_URI` containing percent-escaped characters
|
||||
are now properly decoded (:issue:`3941`)
|
||||
|
||||
* A memory-handling and error-handling issue in
|
||||
:func:`scrapy.utils.ssl.get_temp_key_info` has been fixed (:issue:`3920`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* The documentation now covers how to define and configure a :ref:`custom log
|
||||
format <custom-log-formats>` (:issue:`3616`, :issue:`3660`)
|
||||
|
||||
* API documentation added for :class:`~scrapy.exporters.MarshalItemExporter`
|
||||
and :class:`~scrapy.exporters.PythonItemExporter` (:issue:`3973`)
|
||||
|
||||
* API documentation added for :class:`~scrapy.item.BaseItem` and
|
||||
:class:`~scrapy.item.ItemMeta` (:issue:`3999`)
|
||||
|
||||
* Minor documentation fixes (:issue:`2998`, :issue:`3398`, :issue:`3597`,
|
||||
:issue:`3894`, :issue:`3934`, :issue:`3978`, :issue:`3993`, :issue:`4022`,
|
||||
:issue:`4028`, :issue:`4033`, :issue:`4046`, :issue:`4050`, :issue:`4055`,
|
||||
:issue:`4056`, :issue:`4061`, :issue:`4072`, :issue:`4071`, :issue:`4079`,
|
||||
:issue:`4081`, :issue:`4089`, :issue:`4093`)
|
||||
|
||||
|
||||
.. _1.8-deprecation-removals:
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* ``scrapy.xlib`` has been removed (:issue:`4015`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* The LevelDB_ storage backend
|
||||
(``scrapy.extensions.httpcache.LeveldbCacheStorage``) of
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is
|
||||
deprecated (:issue:`4085`, :issue:`4092`)
|
||||
|
||||
* Use of the undocumented ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
|
||||
variable is deprecated (:issue:`3910`)
|
||||
|
||||
* ``scrapy.item.DictItem`` is deprecated, use :class:`~scrapy.item.Item`
|
||||
instead (:issue:`3999`)
|
||||
|
||||
|
||||
Other changes
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Minimum versions of optional Scrapy requirements that are covered by
|
||||
continuous integration tests have been updated:
|
||||
|
||||
* botocore_ 1.3.23
|
||||
* Pillow_ 3.4.2
|
||||
|
||||
Lower versions of these optional requirements may work, but it is not
|
||||
guaranteed (:issue:`3892`)
|
||||
|
||||
* GitHub templates for bug reports and feature requests (:issue:`3126`,
|
||||
:issue:`3471`, :issue:`3749`, :issue:`3754`)
|
||||
|
||||
* Continuous integration fixes (:issue:`3923`)
|
||||
|
||||
* Code cleanup (:issue:`3391`, :issue:`3907`, :issue:`3946`, :issue:`3950`,
|
||||
:issue:`4023`, :issue:`4031`)
|
||||
|
||||
|
||||
.. _release-1.7.4:
|
||||
|
||||
Scrapy 1.7.4 (2019-10-21)
|
||||
-------------------------
|
||||
|
||||
Revert the fix for :issue:`3804` (:issue:`3819`), which has a few undesired
|
||||
side effects (:issue:`3897`, :issue:`3976`).
|
||||
|
||||
As a result, when an item loader is initialized with an item,
|
||||
:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` once again
|
||||
makes later calls to :meth:`ItemLoader.get_output_value()
|
||||
<scrapy.loader.ItemLoader.get_output_value>` or :meth:`ItemLoader.load_item()
|
||||
<scrapy.loader.ItemLoader.load_item>` return empty data.
|
||||
|
||||
|
||||
.. _release-1.7.3:
|
||||
|
||||
Scrapy 1.7.3 (2019-08-01)
|
||||
-------------------------
|
||||
|
||||
Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
|
||||
|
||||
|
||||
.. _release-1.7.2:
|
||||
|
||||
Scrapy 1.7.2 (2019-07-23)
|
||||
-------------------------
|
||||
|
||||
Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
|
||||
|
||||
|
||||
.. _release-1.7.1:
|
||||
|
||||
Scrapy 1.7.1 (2019-07-18)
|
||||
-------------------------
|
||||
|
||||
Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
|
||||
|
||||
|
||||
.. _release-1.7.0:
|
||||
|
||||
Scrapy 1.7.0 (2019-07-18)
|
||||
|
|
@ -71,7 +295,7 @@ New features
|
|||
~~~~~~~~~~~~
|
||||
|
||||
* A new scheduler priority queue,
|
||||
:class:`scrapy.pqueues.DownloaderAwarePriorityQueue`, may be
|
||||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targetting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
|
@ -84,12 +308,12 @@ New features
|
|||
convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
|
||||
|
||||
* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
|
||||
constructor now receives the :class:`~scrapy.http.Response` object that
|
||||
``__init__`` method now receives the :class:`~scrapy.http.Response` object that
|
||||
originated the request as its second argument (:issue:`3682`)
|
||||
|
||||
* A new ``restrict_text`` parameter for the
|
||||
:attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
constructor allows filtering links by linking text (:issue:`3622`,
|
||||
``__init__`` method allows filtering links by linking text (:issue:`3622`,
|
||||
:issue:`3635`)
|
||||
|
||||
* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
|
||||
|
|
@ -150,9 +374,9 @@ Bug fixes
|
|||
:setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
|
||||
(:issue:`3625`)
|
||||
|
||||
* Fixed a memory leak in :class:`~scrapy.pipelines.media.MediaPipeline`
|
||||
affecting, for example, non-200 responses and exceptions from custom
|
||||
middlewares (:issue:`3813`)
|
||||
* Fixed a memory leak in ``scrapy.pipelines.media.MediaPipeline`` affecting,
|
||||
for example, non-200 responses and exceptions from custom middlewares
|
||||
(:issue:`3813`)
|
||||
|
||||
* Requests with private callbacks are now correctly unserialized from disk
|
||||
(:issue:`3790`)
|
||||
|
|
@ -255,7 +479,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
|
|||
|
||||
* From :class:`~scrapy.selector.Selector`:
|
||||
|
||||
* ``_root`` (both the constructor argument and the object property, use
|
||||
* ``_root`` (both the ``__init__`` method argument and the object property, use
|
||||
``root``)
|
||||
|
||||
* ``extract_unquoted`` (use ``getall``)
|
||||
|
|
@ -301,7 +525,7 @@ Deprecations
|
|||
|
||||
* The ``queuelib.PriorityQueue`` value for the
|
||||
:setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
|
||||
:class:`scrapy.pqueues.ScrapyPriorityQueue` instead.
|
||||
``scrapy.pqueues.ScrapyPriorityQueue`` instead.
|
||||
|
||||
* ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
|
||||
do not accept two arguments are deprecated.
|
||||
|
|
@ -551,12 +775,12 @@ Scrapy 1.5.2 (2019-01-22)
|
|||
|
||||
*The fix is backward incompatible*, it enables telnet user-password
|
||||
authentication by default with a random generated password. If you can't
|
||||
upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT`
|
||||
upgrade right away, please consider setting :setting:`TELNETCONSOLE_PORT`
|
||||
out of its default value.
|
||||
|
||||
See :ref:`telnet console <topics-telnetconsole>` documentation for more info
|
||||
|
||||
* Backport CI build failure under GCE environemnt due to boto import error.
|
||||
* Backport CI build failure under GCE environment due to boto import error.
|
||||
|
||||
.. _release-1.5.1:
|
||||
|
||||
|
|
@ -758,7 +982,9 @@ Enjoy! (Or read on for the rest of changes in this release.)
|
|||
Deprecations and Backward Incompatible Changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
|
||||
- Default to ``canonicalize=False`` in
|
||||
:class:`scrapy.linkextractors.LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
(:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
|
||||
**warning, this is technically backward-incompatible**
|
||||
- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
|
||||
|
|
@ -794,10 +1020,13 @@ New Features
|
|||
- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
|
||||
- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
|
||||
- Warn users when project contains duplicate spider names (fixes :issue:`2181`)
|
||||
- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
|
||||
- :ref:`Media downloads <topics-media-pipeline>`, with :class:`FilesPipelines`
|
||||
or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
|
||||
using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
|
||||
- ``scrapy.utils.datatypes.CaselessDict`` now accepts ``Mapping`` instances and
|
||||
not only dicts (:issue:`2646`)
|
||||
- :ref:`Media downloads <topics-media-pipeline>`, with
|
||||
:class:`~scrapy.pipelines.files.FilesPipeline` or
|
||||
:class:`~scrapy.pipelines.images.ImagesPipeline`, can now optionally handle
|
||||
HTTP redirects using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting
|
||||
(:issue:`2616`, fixes :issue:`2004`)
|
||||
- Accept non-complete responses from websites using a new
|
||||
:setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
|
||||
- Optional pretty-printing of JSON and XML items via
|
||||
|
|
@ -817,8 +1046,8 @@ Bug fixes
|
|||
|
||||
- LinkExtractor now strips leading and trailing whitespaces from attributes
|
||||
(:issue:`2547`, fixes :issue:`1614`)
|
||||
- Properly handle whitespaces in action attribute in :class:`FormRequest`
|
||||
(:issue:`2548`)
|
||||
- Properly handle whitespaces in action attribute in
|
||||
:class:`~scrapy.http.FormRequest` (:issue:`2548`)
|
||||
- Buffer CONNECT response bytes from proxy until all HTTP headers are received
|
||||
(:issue:`2495`, fixes :issue:`2491`)
|
||||
- FTP downloader now works on Python 3, provided you use Twisted>=17.1
|
||||
|
|
@ -851,7 +1080,8 @@ Cleanups & Refactoring
|
|||
fixes :issue:`2560`)
|
||||
- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
|
||||
- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
|
||||
- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
|
||||
- Catch more specific ``os.error`` exception in
|
||||
``scrapy.pipelines.files.FSFilesStore`` (:issue:`2644`)
|
||||
- Change "localhost" test server certificate (:issue:`2720`)
|
||||
- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
|
||||
|
||||
|
|
@ -868,7 +1098,8 @@ Documentation
|
|||
(:issue:`2477`, fixes :issue:`2475`)
|
||||
- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
|
||||
- Rearrange selector sections (:issue:`2705`)
|
||||
- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
|
||||
- Remove ``__nonzero__`` from :class:`~scrapy.selector.SelectorList`
|
||||
docs (:issue:`2683`)
|
||||
- Mention how to disable request filtering in documentation of
|
||||
:setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
|
||||
- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
|
||||
|
|
@ -2327,7 +2558,7 @@ Scrapy 0.18.0 (released 2013-08-09)
|
|||
- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
|
||||
- Add scrapy commands using external libraries (:issue:`260`)
|
||||
- Added ``--pdb`` option to ``scrapy`` command line tool
|
||||
- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
|
||||
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
|
||||
- Several improvements to spider contracts
|
||||
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
|
||||
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
|
||||
|
|
@ -2448,7 +2679,7 @@ Scrapy changes:
|
|||
- added options ``-o`` and ``-t`` to the :command:`runspider` command
|
||||
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
|
||||
- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
|
||||
- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
|
||||
- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
|
||||
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
- dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
|
||||
|
|
@ -2472,7 +2703,7 @@ Scrapy changes:
|
|||
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
|
||||
- promoted :ref:`topics-djangoitem` to main contrib
|
||||
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
|
||||
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
|
||||
- removed signal: ``scrapy.mail.mail_sent``
|
||||
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
|
||||
|
|
@ -2590,7 +2821,7 @@ Code rearranged and removed
|
|||
- `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`)
|
||||
- `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`)
|
||||
- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`)
|
||||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on github: https://github.com/scrapy/dirbot
|
||||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
|
|
@ -2609,7 +2840,8 @@ The numbers like #NNN reference tickets in the old issue tracker (Trac) which is
|
|||
New features and improvements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273)
|
||||
- Passed item is now sent in the ``item`` argument of the :signal:`item_passed
|
||||
<item_scraped>` (#273)
|
||||
- Added verbose option to ``scrapy version`` command, useful for bug reports (#298)
|
||||
- HTTP cache now stored by default in the project data dir (#279)
|
||||
- Added project data storage directory (#276, #277)
|
||||
|
|
@ -2685,7 +2917,7 @@ API changes
|
|||
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
|
||||
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
|
||||
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
|
||||
- Changes to Scrapy Manager (now called "Crawler"):
|
||||
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
|
||||
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
|
||||
|
|
@ -2810,23 +3042,35 @@ First release of Scrapy.
|
|||
|
||||
|
||||
.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
|
||||
.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
|
||||
.. _cryptography: https://cryptography.io/en/latest/
|
||||
.. _cssselect: https://github.com/scrapy/cssselect/
|
||||
.. _docstrings: https://docs.python.org/glossary.html#term-docstring
|
||||
.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
|
||||
.. _LevelDB: https://github.com/google/leveldb
|
||||
.. _lxml: http://lxml.de/
|
||||
.. _marshal: https://docs.python.org/2/library/marshal.html
|
||||
.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
|
||||
.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
|
||||
.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
|
||||
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
|
||||
.. _Pillow: https://python-pillow.org/
|
||||
.. _pyOpenSSL: https://www.pyopenssl.org/en/stable/
|
||||
.. _queuelib: https://github.com/scrapy/queuelib
|
||||
.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
|
||||
.. _resource: https://docs.python.org/2/library/resource.html
|
||||
.. _robots.txt: http://www.robotstxt.org/
|
||||
.. _scrapely: https://github.com/scrapy/scrapely
|
||||
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
|
||||
.. _six: https://six.readthedocs.io/
|
||||
.. _tox: https://pypi.python.org/pypi/tox
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _w3lib: https://github.com/scrapy/w3lib
|
||||
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
|
||||
.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
|
||||
.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
|
||||
.. _Zsh: https://www.zsh.org/
|
||||
|
|
|
|||
|
|
@ -1,2 +1,5 @@
|
|||
-r ../requirements-py3.txt
|
||||
Sphinx>=2.1
|
||||
sphinx_rtd_theme
|
||||
sphinx-hoverxref
|
||||
sphinx-notfound-page
|
||||
sphinx_rtd_theme
|
||||
|
|
|
|||
|
|
@ -273,5 +273,3 @@ class (which they all inherit from).
|
|||
|
||||
Close the given spider. After this is called, no more specific stats
|
||||
can be accessed or collected.
|
||||
|
||||
.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
|
|
|
|||
|
|
@ -166,11 +166,10 @@ for concurrency.
|
|||
For more information about asynchronous programming and Twisted see these
|
||||
links:
|
||||
|
||||
* `Introduction to Deferreds in Twisted`_
|
||||
* :doc:`twisted:core/howto/defer-intro`
|
||||
* `Twisted - hello, asynchronous programming`_
|
||||
* `Twisted Introduction - Krondo`_
|
||||
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. highlight:: none
|
||||
|
||||
.. _topics-commands:
|
||||
|
||||
=================
|
||||
|
|
@ -66,7 +68,9 @@ structure by default, similar to this::
|
|||
|
||||
The directory where the ``scrapy.cfg`` file resides is known as the *project
|
||||
root directory*. That file contains the name of the python module that defines
|
||||
the project settings. Here is an example::
|
||||
the project settings. Here is an example:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[settings]
|
||||
default = myproject.settings
|
||||
|
|
@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
|
|||
shared by multiple Scrapy projects, each with its own settings module.
|
||||
|
||||
In that case, you must define one or more aliases for those settings modules
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file::
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[settings]
|
||||
default = myproject1.settings
|
||||
|
|
@ -277,6 +283,8 @@ check
|
|||
|
||||
Run contract checks.
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage examples::
|
||||
|
||||
$ scrapy check -l
|
||||
|
|
@ -294,6 +302,8 @@ Usage examples::
|
|||
[FAILED] first_spider:parse
|
||||
>>> Returned 92 requests, expected 0..4
|
||||
|
||||
.. skip: end
|
||||
|
||||
.. command:: list
|
||||
|
||||
list
|
||||
|
|
@ -481,6 +491,8 @@ Supported options:
|
|||
|
||||
* ``--verbose`` or ``-v``: display information for each depth level
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage example::
|
||||
|
||||
$ scrapy parse http://www.example.com/ -c parse_item
|
||||
|
|
@ -495,6 +507,8 @@ Usage example::
|
|||
# Requests -----------------------------------------------------------------
|
||||
[]
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
||||
.. command:: settings
|
||||
|
||||
|
|
@ -573,7 +587,9 @@ Default: ``''`` (empty string)
|
|||
A module to use for looking up custom Scrapy commands. This is used to add custom
|
||||
commands for your Scrapy project.
|
||||
|
||||
Example::
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
COMMANDS_MODULE = 'mybot.commands'
|
||||
|
||||
|
|
@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
|
|||
``scrapy.commands`` section in the entry points of the library ``setup.py``
|
||||
file.
|
||||
|
||||
The following example adds ``my_command`` command::
|
||||
The following example adds ``my_command`` command:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
|
|
|
|||
|
|
@ -6,10 +6,6 @@ Spiders Contracts
|
|||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
|
||||
to minor functionality/API updates. Check the :ref:`release notes <news>` to
|
||||
be notified of updates.
|
||||
|
||||
Testing spiders can get particularly annoying and while nothing prevents you
|
||||
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
|
||||
integrated way of testing your spiders by the means of contracts.
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
|
|||
of the spider at the method level. It has the advantage of being flexible and
|
||||
simple to use, but does not allow debugging code inside a method.
|
||||
|
||||
.. highlight:: none
|
||||
|
||||
.. skip: start
|
||||
|
||||
In order to see the item scraped from a specific url::
|
||||
|
||||
$ scrapy parse --spider=myspider -c parse_item -d 2 <item_url>
|
||||
|
|
@ -85,6 +89,8 @@ using::
|
|||
|
||||
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
||||
Scrapy Shell
|
||||
============
|
||||
|
|
@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides
|
|||
showing the response received and the output. How to debug the situation when
|
||||
``parse_details`` sometimes receives no item?
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
Fortunately, the :command:`shell` is your bread and butter in this case (see
|
||||
:ref:`topics-shell-inspect-response`)::
|
||||
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ where our quotes are coming from:
|
|||
First click on the request with the name ``scroll``. On the right
|
||||
you can now inspect the request. In ``Headers`` you'll find details
|
||||
about the request headers, such as the URL, the method, the IP-address,
|
||||
and so on. We'll ignore the other tabs and click directly on ``Reponse``.
|
||||
and so on. We'll ignore the other tabs and click directly on ``Response``.
|
||||
|
||||
What you should see in the ``Preview`` pane is the rendered HTML-code,
|
||||
that is exactly what we saw when we called ``view(response)`` in the
|
||||
|
|
|
|||
|
|
@ -348,7 +348,6 @@ HttpCacheMiddleware
|
|||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
* :ref:`httpcache-storage-leveldb`
|
||||
|
||||
You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
|
||||
setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
|
||||
|
|
@ -475,30 +474,9 @@ DBM storage backend
|
|||
|
||||
A DBM_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
By default, it uses the anydbm_ module, but you can change it with the
|
||||
By default, it uses the :mod:`dbm`, but you can change it with the
|
||||
:setting:`HTTPCACHE_DBM_MODULE` setting.
|
||||
|
||||
.. _httpcache-storage-leveldb:
|
||||
|
||||
LevelDB storage backend
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. class:: LeveldbCacheStorage
|
||||
|
||||
.. versionadded:: 0.23
|
||||
|
||||
A LevelDB_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
This backend is not recommended for development because only one process
|
||||
can access LevelDB databases at the same time, so you can't run a crawl and
|
||||
open the scrapy shell in parallel for the same spider.
|
||||
|
||||
In order to use this storage backend, install the `LevelDB python
|
||||
bindings`_ (e.g. ``pip install leveldb``).
|
||||
|
||||
.. _LevelDB: https://github.com/google/leveldb
|
||||
.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
|
||||
|
||||
.. _httpcache-storage-custom:
|
||||
|
||||
Writing your own storage backend
|
||||
|
|
@ -513,7 +491,7 @@ defines the methods described below.
|
|||
|
||||
.. method:: open_spider(spider)
|
||||
|
||||
This method gets called after a spider has been opened for crawling. It handles
|
||||
This method gets called after a spider has been opened for crawling. It handles
|
||||
the :signal:`open_spider <spider_opened>` signal.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
|
|
@ -521,8 +499,8 @@ defines the methods described below.
|
|||
|
||||
.. method:: close_spider(spider)
|
||||
|
||||
This method gets called after a spider has been closed. It handles
|
||||
the :signal:`close_spider <spider_closed>` signal.
|
||||
This method gets called after a spider has been closed. It handles
|
||||
the :signal:`close_spider <spider_closed>` signal.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -534,7 +512,7 @@ defines the methods described below.
|
|||
:param spider: the spider which generated the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
:param request: the request to find cached reponse for
|
||||
:param request: the request to find cached response for
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
||||
.. method:: store_response(spider, request, response)
|
||||
|
|
@ -648,7 +626,7 @@ HTTPCACHE_DBM_MODULE
|
|||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``'anydbm'``
|
||||
Default: ``'dbm'``
|
||||
|
||||
The database module to use in the :ref:`DBM storage backend
|
||||
<httpcache-storage-dbm>`. This setting is specific to the DBM backend.
|
||||
|
|
@ -1000,10 +978,10 @@ RobotsTxtMiddleware
|
|||
|
||||
Scrapy ships with support for the following robots.txt_ parsers:
|
||||
|
||||
* :ref:`RobotFileParser <python-robotfileparser>` (default)
|
||||
* :ref:`Protego <protego-parser>` (default)
|
||||
* :ref:`RobotFileParser <python-robotfileparser>`
|
||||
* :ref:`Reppy <reppy-parser>`
|
||||
* :ref:`Robotexclusionrulesparser <rerp-parser>`
|
||||
* :ref:`Protego <protego-parser>`
|
||||
|
||||
You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
|
||||
setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
|
||||
|
|
@ -1015,50 +993,81 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
|
|||
the request will be ignored by this middleware even if
|
||||
:setting:`ROBOTSTXT_OBEY` is enabled.
|
||||
|
||||
Parsers vary in several aspects:
|
||||
|
||||
* Language of implementation
|
||||
|
||||
* Supported specification
|
||||
|
||||
* Support for wildcard matching
|
||||
|
||||
* Usage of `length based rule <https://developers.google.com/search/reference/robots_txt#order-of-precedence-for-group-member-lines>`_:
|
||||
in particular for ``Allow`` and ``Disallow`` directives, where the most
|
||||
specific rule based on the length of the path trumps the less specific
|
||||
(shorter) rule
|
||||
|
||||
Performance comparison of different parsers is available at `the following link
|
||||
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
|
||||
|
||||
.. _protego-parser:
|
||||
|
||||
Protego parser
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
Based on `Protego <https://github.com/scrapy/protego>`_:
|
||||
|
||||
* implemented in Python
|
||||
|
||||
* is compliant with `Google's Robots.txt Specification
|
||||
<https://developers.google.com/search/reference/robots_txt>`_
|
||||
|
||||
* supports wildcard matching
|
||||
|
||||
* uses the length based rule
|
||||
|
||||
Scrapy uses this parser by default.
|
||||
|
||||
.. _python-robotfileparser:
|
||||
|
||||
RobotFileParser
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is
|
||||
Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's
|
||||
1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
|
||||
support for wildcard matching. Scrapy uses this parser by default.
|
||||
Based on `RobotFileParser
|
||||
<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
|
||||
|
||||
* is Python's built-in robots.txt_ parser
|
||||
|
||||
* is compliant with `Martijn Koster's 1996 draft specification
|
||||
<http://www.robotstxt.org/norobots-rfc.txt>`_
|
||||
|
||||
* lacks support for wildcard matching
|
||||
|
||||
* doesn't use the length based rule
|
||||
|
||||
It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
|
||||
|
||||
In order to use this parser, set:
|
||||
|
||||
* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
|
||||
|
||||
.. _rerp-parser:
|
||||
|
||||
Robotexclusionrulesparser
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ is fully compliant
|
||||
with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
|
||||
with support for wildcard matching.
|
||||
|
||||
In order to use this parser:
|
||||
|
||||
* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
|
||||
``pip install robotexclusionrulesparser``
|
||||
|
||||
* Set :setting:`ROBOTSTXT_PARSER` setting to
|
||||
``scrapy.robotstxt.RerpRobotParser``
|
||||
|
||||
.. _reppy-parser:
|
||||
|
||||
Reppy parser
|
||||
~~~~~~~~~~~~
|
||||
|
||||
`Reppy <https://github.com/seomoz/reppy/>`_ is a Python wrapper around `Robots Exclusion
|
||||
Protocol Parser for C++ <https://github.com/seomoz/rep-cpp>`_. The parser is fully compliant
|
||||
with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
|
||||
with support for wildcard matching. Unlike
|
||||
`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ and
|
||||
`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_, it uses the length based
|
||||
rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific
|
||||
rule based on the length of the path trumps the less specific (shorter) rule.
|
||||
Based on `Reppy <https://github.com/seomoz/reppy/>`_:
|
||||
|
||||
* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
|
||||
<https://github.com/seomoz/rep-cpp>`_
|
||||
|
||||
* is compliant with `Martijn Koster's 1996 draft specification
|
||||
<http://www.robotstxt.org/norobots-rfc.txt>`_
|
||||
|
||||
* supports wildcard matching
|
||||
|
||||
* uses the length based rule
|
||||
|
||||
Native implementation, provides better speed than Protego.
|
||||
|
||||
In order to use this parser:
|
||||
|
||||
|
|
@ -1067,22 +1076,29 @@ In order to use this parser:
|
|||
* Set :setting:`ROBOTSTXT_PARSER` setting to
|
||||
``scrapy.robotstxt.ReppyRobotParser``
|
||||
|
||||
.. _protego-parser:
|
||||
.. _rerp-parser:
|
||||
|
||||
Protego parser
|
||||
~~~~~~~~~~~~~~
|
||||
Robotexclusionrulesparser
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
`Protego <https://github.com/scrapy/protego>`_ is a pure-Python robots.txt_ parser.
|
||||
The parser is fully compliant with `Google's Robots.txt Specification
|
||||
<https://developers.google.com/search/reference/robots_txt>`_ hence supports wildcard
|
||||
matching, and uses the length based rule similar to `Reppy <https://github.com/seomoz/reppy/>`_.
|
||||
Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
|
||||
|
||||
* implemented in Python
|
||||
|
||||
* is compliant with `Martijn Koster's 1996 draft specification
|
||||
<http://www.robotstxt.org/norobots-rfc.txt>`_
|
||||
|
||||
* supports wildcard matching
|
||||
|
||||
* doesn't use the length based rule
|
||||
|
||||
In order to use this parser:
|
||||
|
||||
* Install `Protego <https://github.com/scrapy/protego>`_ by running ``pip install protego``
|
||||
* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
|
||||
``pip install robotexclusionrulesparser``
|
||||
|
||||
* Set :setting:`ROBOTSTXT_PARSER` setting to
|
||||
``scrapy.robotstxt.ProtegoRobotParser``
|
||||
``scrapy.robotstxt.RerpRobotParser``
|
||||
|
||||
.. _support-for-new-robots-parser:
|
||||
|
||||
|
|
@ -1186,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
|
|||
|
||||
|
||||
.. _DBM: https://en.wikipedia.org/wiki/Dbm
|
||||
.. _anydbm: https://docs.python.org/2/library/anydbm.html
|
||||
|
|
|
|||
|
|
@ -9,19 +9,19 @@ Sending e-mail
|
|||
|
||||
Although Python makes sending e-mails relatively easy via the `smtplib`_
|
||||
library, Scrapy provides its own facility for sending e-mails which is very
|
||||
easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
|
||||
interfering with the non-blocking IO of the crawler. It also provides a
|
||||
simple API for sending attachments and it's very easy to configure, with a few
|
||||
:ref:`settings <topics-email-settings>`.
|
||||
easy to use and it's implemented using :doc:`Twisted non-blocking IO
|
||||
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
|
||||
IO of the crawler. It also provides a simple API for sending attachments and
|
||||
it's very easy to configure, with a few :ref:`settings
|
||||
<topics-email-settings>`.
|
||||
|
||||
.. _smtplib: https://docs.python.org/2/library/smtplib.html
|
||||
.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
|
||||
|
||||
Quick example
|
||||
=============
|
||||
|
||||
There are two ways to instantiate the mail sender. You can instantiate it using
|
||||
the standard constructor::
|
||||
the standard ``__init__`` method::
|
||||
|
||||
from scrapy.mail import MailSender
|
||||
mailer = MailSender()
|
||||
|
|
@ -39,7 +39,8 @@ MailSender class reference
|
|||
==========================
|
||||
|
||||
MailSender is the preferred class to use for sending emails from Scrapy, as it
|
||||
uses `Twisted non-blocking IO`_, like the rest of the framework.
|
||||
uses :doc:`Twisted non-blocking IO <twisted:core/howto/defer-intro>`, like the
|
||||
rest of the framework.
|
||||
|
||||
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
|
||||
|
||||
|
|
@ -111,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
|
|||
Mail settings
|
||||
=============
|
||||
|
||||
These settings define the default constructor values of the :class:`MailSender`
|
||||
These settings define the default ``__init__`` method values of the :class:`MailSender`
|
||||
class, and can be used to configure e-mail notifications in your project without
|
||||
writing any code (for those extensions and code that uses :class:`MailSender`).
|
||||
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ described next.
|
|||
1. Declaring a serializer in the field
|
||||
--------------------------------------
|
||||
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
a callable which receives a value and returns its serialized form.
|
||||
|
||||
Example::
|
||||
|
|
@ -144,7 +144,7 @@ BaseItemExporter
|
|||
defining what fields to export, whether to export empty fields, or which
|
||||
encoding to use.
|
||||
|
||||
These features can be configured through the constructor arguments which
|
||||
These features can be configured through the ``__init__`` method arguments which
|
||||
populate their respective instance attributes: :attr:`fields_to_export`,
|
||||
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
|
||||
|
||||
|
|
@ -246,8 +246,8 @@ XmlItemExporter
|
|||
:param item_element: The name of each item element in the exported XML.
|
||||
:type item_element: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -306,9 +306,9 @@ CsvItemExporter
|
|||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor, and the leftover arguments to the
|
||||
`csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
|
||||
argument to customize this exporter.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
|
@ -334,8 +334,8 @@ PickleItemExporter
|
|||
|
||||
For more information, refer to the `pickle module documentation`_.
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
Pickle isn't a human readable format, so no output examples are provided.
|
||||
|
||||
|
|
@ -351,8 +351,8 @@ PprintItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -367,10 +367,10 @@ JsonItemExporter
|
|||
.. class:: JsonItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional constructor arguments are
|
||||
passed to the :class:`BaseItemExporter` constructor, and the leftover
|
||||
arguments to the `JSONEncoder`_ constructor, so you can use any
|
||||
`JSONEncoder`_ constructor argument to customize this exporter.
|
||||
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
|
||||
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -398,10 +398,10 @@ JsonLinesItemExporter
|
|||
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional constructor arguments are passed
|
||||
to the :class:`BaseItemExporter` constructor, and the leftover arguments to
|
||||
the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
|
||||
constructor argument to customize this exporter.
|
||||
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
|
||||
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
|
||||
``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ Loading & activating extensions
|
|||
|
||||
Extensions are loaded and activated at startup by instantiating a single
|
||||
instance of the extension class. Therefore, all the extension initialization
|
||||
code must be performed in the class constructor (``__init__`` method).
|
||||
code must be performed in the class ``__init__`` method.
|
||||
|
||||
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
|
||||
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
|
||||
|
|
|
|||
|
|
@ -99,12 +99,12 @@ The storages backends supported out of the box are:
|
|||
|
||||
* :ref:`topics-feed-storage-fs`
|
||||
* :ref:`topics-feed-storage-ftp`
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
* :ref:`topics-feed-storage-stdout`
|
||||
|
||||
Some storage backends may be unavailable if the required external libraries are
|
||||
not available. For example, the S3 backend is only available if the botocore_
|
||||
or boto_ library is installed (Scrapy supports boto_ only on Python 2).
|
||||
library is installed.
|
||||
|
||||
|
||||
.. _topics-feed-uri-params:
|
||||
|
|
@ -182,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
|
|||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
|
||||
* Required external libraries: `botocore`_
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
|
@ -399,6 +399,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
|
||||
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _boto: https://github.com/boto/boto
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
|
|||
|
||||
This method is called for every item pipeline component. :meth:`process_item`
|
||||
must either: return a dict with data, return an :class:`~scrapy.item.Item`
|
||||
(or any descendant class) object, return a `Twisted Deferred`_ or raise
|
||||
(or any descendant class) object, return a
|
||||
:class:`~twisted.internet.defer.Deferred` or raise
|
||||
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
|
||||
processed by further pipeline components.
|
||||
|
||||
|
|
@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
|
|||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
||||
.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
Item pipeline example
|
||||
=====================
|
||||
|
||||
|
|
@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
|
|||
Take screenshot of item
|
||||
-----------------------
|
||||
|
||||
This example demonstrates how to return Deferred_ from :meth:`process_item` method.
|
||||
This example demonstrates how to return a
|
||||
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
|
||||
It uses Splash_ to render screenshot of item url. Pipeline
|
||||
makes request to locally running instance of Splash_. After request is downloaded
|
||||
and Deferred callback fires, it saves item to a file and adds filename to an item.
|
||||
|
|
@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
return item
|
||||
|
||||
.. _Splash: https://splash.readthedocs.io/en/stable/
|
||||
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
Duplicates filter
|
||||
-----------------
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ especially in a larger project with many spiders.
|
|||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide a `dictionary-like`_ API with a convenient syntax for declaring
|
||||
their available fields.
|
||||
their available fields.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
serialization can be customized using Item fields metadata, :mod:`trackref`
|
||||
tracks Item instances to help find memory leaks
|
||||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
|
||||
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
|
@ -237,8 +237,12 @@ Item objects
|
|||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
|
||||
Items replicate the standard `dict API`_, including its constructor. The
|
||||
only additional attribute provided by Items is:
|
||||
Items replicate the standard `dict API`_, including its ``__init__`` method, and
|
||||
also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
||||
.. automethod:: deepcopy
|
||||
|
||||
.. attribute:: fields
|
||||
|
||||
|
|
|
|||
|
|
@ -71,34 +71,11 @@ on cookies.
|
|||
Request serialization
|
||||
---------------------
|
||||
|
||||
Requests must be serializable by the ``pickle`` module, in order for persistence
|
||||
to work, so you should make sure that your requests are serializable.
|
||||
|
||||
The most common issue here is to use ``lambda`` functions on request callbacks that
|
||||
can't be persisted.
|
||||
|
||||
So, for example, this won't work::
|
||||
|
||||
def some_callback(self, response):
|
||||
somearg = 'test'
|
||||
return scrapy.Request('http://www.example.com',
|
||||
callback=lambda r: self.other_callback(r, somearg))
|
||||
|
||||
def other_callback(self, response, somearg):
|
||||
print("the argument passed is: %s" % somearg)
|
||||
|
||||
But this will::
|
||||
|
||||
def some_callback(self, response):
|
||||
somearg = 'test'
|
||||
return scrapy.Request('http://www.example.com',
|
||||
callback=self.other_callback, cb_kwargs={'somearg': somearg})
|
||||
|
||||
def other_callback(self, response, somearg):
|
||||
print("the argument passed is: %s" % somearg)
|
||||
For persistence to work, :class:`~scrapy.http.Request` objects must be
|
||||
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
|
||||
values passed to their ``__init__`` method, which must be methods of the
|
||||
runnning :class:`~scrapy.spiders.Spider` class.
|
||||
|
||||
If you wish to log the requests that couldn't be serialized, you can set the
|
||||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||
It is ``False`` by default.
|
||||
|
||||
.. _pickle: https://docs.python.org/library/pickle.html
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ knowledge about Python internals. For more info about Guppy, refer to the
|
|||
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
If you're using Python 3, you can use muppy from `Pympler`_.
|
||||
You can use muppy from `Pympler`_.
|
||||
|
||||
.. _Pympler: https://pypi.org/project/Pympler/
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Using Item Loaders to populate items
|
|||
|
||||
To use an Item Loader, you must first instantiate it. You can either
|
||||
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
|
||||
which case an Item is automatically instantiated in the Item Loader constructor
|
||||
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
|
||||
using the Item class specified in the :attr:`ItemLoader.default_item_class`
|
||||
attribute.
|
||||
|
||||
|
|
@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
|
|||
the same item field; the Item Loader will know how to "join" those values later
|
||||
using a proper processing function.
|
||||
|
||||
.. note:: Collected data is internally stored as lists,
|
||||
allowing to add several values to the same field.
|
||||
If an ``item`` argument is passed when creating a loader,
|
||||
each of the item's values will be stored as-is if it's already
|
||||
an iterable, or wrapped with a list if it's a single value.
|
||||
|
||||
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
|
||||
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
|
||||
chapter <topics-items>`::
|
||||
|
|
@ -128,28 +134,14 @@ So what happens is:
|
|||
It's worth noticing that processors are just callable objects, which are called
|
||||
with the data to be parsed, and return a parsed value. So you can use any
|
||||
function as input or output processor. The only requirement is that they must
|
||||
accept one (and only one) positional argument, which will be an iterator.
|
||||
accept one (and only one) positional argument, which will be an iterable.
|
||||
|
||||
.. note:: Both input and output processors must receive an iterator as their
|
||||
.. note:: Both input and output processors must receive an iterable as their
|
||||
first argument. The output of those functions can be anything. The result of
|
||||
input processors will be appended to an internal list (in the Loader)
|
||||
containing the collected values (for that field). The result of the output
|
||||
processors is the value that will be finally assigned to the item.
|
||||
|
||||
If you want to use a plain function as a processor, make sure it receives
|
||||
``self`` as the first argument::
|
||||
|
||||
def lowercase_processor(self, values):
|
||||
for v in values:
|
||||
yield v.lower()
|
||||
|
||||
class MyItemLoader(ItemLoader):
|
||||
name_in = lowercase_processor
|
||||
|
||||
This is because whenever a function is assigned as a class variable, it becomes
|
||||
a method and would be passed the instance as the the first argument when being
|
||||
called. See `this answer on stackoverflow`_ for more details.
|
||||
|
||||
The other thing you need to keep in mind is that the values returned by input
|
||||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
|
@ -157,7 +149,7 @@ processors to populate the fields.
|
|||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
|
||||
.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
|
||||
|
||||
|
||||
Declaring Item Loaders
|
||||
======================
|
||||
|
|
@ -265,7 +257,7 @@ There are several ways to modify Item Loader context values:
|
|||
loader.context['unit'] = 'cm'
|
||||
|
||||
2. On Item Loader instantiation (the keyword arguments of Item Loader
|
||||
constructor are stored in the Item Loader context)::
|
||||
``__init__`` method are stored in the Item Loader context)::
|
||||
|
||||
loader = ItemLoader(product, unit='cm')
|
||||
|
||||
|
|
@ -485,6 +477,8 @@ ItemLoader objects
|
|||
.. attribute:: item
|
||||
|
||||
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
|
||||
This is mostly used as a property so when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
|
|
@ -494,7 +488,7 @@ ItemLoader objects
|
|||
.. attribute:: default_item_class
|
||||
|
||||
An Item class (or factory), used to instantiate items when not given in
|
||||
the constructor.
|
||||
the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
|
|
@ -509,15 +503,15 @@ ItemLoader objects
|
|||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the constructor.
|
||||
If a selector is given in the constructor this attribute is ignored.
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the constructor or one created from
|
||||
the response given in the constructor using the
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
|
||||
|
|
@ -642,7 +636,7 @@ Here is a list of all built-in processors:
|
|||
.. class:: Identity
|
||||
|
||||
The simplest processor, which doesn't do anything. It returns the original
|
||||
values unchanged. It doesn't receive any constructor arguments, nor does it
|
||||
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
|
||||
accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
|
@ -656,7 +650,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
Returns the first non-null/non-empty value from the values received,
|
||||
so it's typically used as an output processor to single-valued fields.
|
||||
It doesn't receive any constructor arguments, nor does it accept Loader contexts.
|
||||
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -667,7 +661,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
.. class:: Join(separator=u' ')
|
||||
|
||||
Returns the values joined with the separator given in the constructor, which
|
||||
Returns the values joined with the separator given in the ``__init__`` method, which
|
||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||
|
||||
When using the default separator, this processor is equivalent to the
|
||||
|
|
@ -705,7 +699,7 @@ Here is a list of all built-in processors:
|
|||
those which do, this processor will pass the currently active :ref:`Loader
|
||||
context <topics-loaders-context>` through that parameter.
|
||||
|
||||
The keyword arguments passed in the constructor are used as the default
|
||||
The keyword arguments passed in the ``__init__`` method are used as the default
|
||||
Loader context values passed to each function call. However, the final
|
||||
Loader context values passed to functions are overridden with the currently
|
||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
|
|
@ -749,12 +743,12 @@ Here is a list of all built-in processors:
|
|||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
constructor keyword arguments are used as default context values. See
|
||||
``__init__`` method keyword arguments are used as default context values. See
|
||||
:class:`Compose` processor for more info.
|
||||
|
||||
.. class:: SelectJmes(json_path)
|
||||
|
||||
Queries the value using the json path provided to the constructor and returns the output.
|
||||
Queries the value using the json path provided to the ``__init__`` method and returns the output.
|
||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||
This processor takes only one input at a time.
|
||||
|
||||
|
|
|
|||
|
|
@ -198,8 +198,9 @@ to override some of the Scrapy settings regarding logging.
|
|||
Custom Log Formats
|
||||
------------------
|
||||
|
||||
A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
|
||||
and making :setting:`LOG_FORMATTER` point to your new class.
|
||||
A custom log format can be set for different actions by extending
|
||||
:class:`~scrapy.logformatter.LogFormatter` class and making
|
||||
:setting:`LOG_FORMATTER` point to your new class.
|
||||
|
||||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
|
@ -254,18 +255,18 @@ scrapy.utils.log module
|
|||
when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
|
||||
In that case, its usage is not required but it's recommended.
|
||||
|
||||
If you plan on configuring the handlers yourself is still recommended you
|
||||
call this function, passing ``install_root_handler=False``. Bear in mind
|
||||
there won't be any log output set by default in that case.
|
||||
Another option when running custom scripts is to manually configure the logging.
|
||||
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
|
||||
|
||||
To get you started on manually configuring logging's output, you can use
|
||||
`logging.basicConfig()`_ to set a basic root handler. This is an example
|
||||
on how to redirect ``INFO`` or higher messages to a file::
|
||||
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
|
||||
so it is recommended to only use `logging.basicConfig()`_ together with
|
||||
:class:`~scrapy.crawler.CrawlerRunner`.
|
||||
|
||||
This is an example on how to redirect ``INFO`` or higher messages to a file::
|
||||
|
||||
import logging
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
configure_logging(install_root_handler=False)
|
||||
logging.basicConfig(
|
||||
filename='log.txt',
|
||||
format='%(levelname)s: %(message)s',
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ policy::
|
|||
|
||||
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
|
||||
|
||||
Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
|
||||
|
||||
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
|
||||
|
|
@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
* ``success`` is a boolean which is ``True`` if the image was downloaded
|
||||
successfully or ``False`` if it failed for some reason
|
||||
|
||||
* ``file_info_or_error`` is a dict containing the following keys (if success
|
||||
is ``True``) or a `Twisted Failure`_ if there was a problem.
|
||||
* ``file_info_or_error`` is a dict containing the following keys (if
|
||||
success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
|
||||
there was a problem.
|
||||
|
||||
* ``url`` - the url where the file was downloaded from. This is the url of
|
||||
the request returned from the :meth:`~get_media_requests`
|
||||
|
|
@ -577,5 +578,4 @@ above::
|
|||
item['image_paths'] = image_paths
|
||||
return item
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running.
|
|||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run() # the script will block here until the crawling is finished
|
||||
|
||||
.. seealso:: `Twisted Reactor Overview`_.
|
||||
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
|
||||
|
||||
.. _run-multiple-spiders:
|
||||
|
||||
|
|
@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
.. _Crawlera: https://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
|
|||
Request objects
|
||||
===============
|
||||
|
||||
.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
|
||||
.. autoclass:: Request
|
||||
|
||||
A :class:`Request` object represents an HTTP request, which is usually
|
||||
generated in the Spider and executed by the Downloader, and thus generating
|
||||
|
|
@ -121,8 +121,8 @@ Request objects
|
|||
|
||||
:param errback: a function that will be called if any exception was
|
||||
raised while processing the request. This includes pages that failed
|
||||
with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
|
||||
as first parameter.
|
||||
with 404 HTTP errors and such. It receives a
|
||||
:exc:`~twisted.python.failure.Failure` as first parameter.
|
||||
For more information,
|
||||
see :ref:`topics-request-response-ref-errbacks` below.
|
||||
:type errback: callable
|
||||
|
|
@ -137,7 +137,7 @@ Request objects
|
|||
|
||||
A string containing the URL of this request. Keep in mind that this
|
||||
attribute contains the escaped URL, so it can differ from the URL passed in
|
||||
the constructor.
|
||||
the ``__init__`` method.
|
||||
|
||||
This attribute is read-only. To change the URL of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing
|
|||
The errback of a request is a function that will be called when an exception
|
||||
is raise while processing it.
|
||||
|
||||
It receives a `Twisted Failure`_ instance as first parameter and can be
|
||||
used to track connection establishment timeouts, DNS errors etc.
|
||||
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
|
||||
be used to track connection establishment timeouts, DNS errors etc.
|
||||
|
||||
Here's an example spider logging all errors and catching some specific
|
||||
errors if needed::
|
||||
|
|
@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
.. class:: FormRequest(url, [formdata, ...])
|
||||
|
||||
The :class:`FormRequest` class adds a new argument to the constructor. The
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
|
|||
:type dont_click: boolean
|
||||
|
||||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` constructor.
|
||||
:class:`FormRequest` ``__init__`` method.
|
||||
|
||||
.. versionadded:: 0.10.3
|
||||
The ``formname`` parameter.
|
||||
|
|
@ -547,7 +547,7 @@ dealing with JSON requests.
|
|||
|
||||
.. class:: JsonRequest(url, [... data, dumps_kwargs])
|
||||
|
||||
The :class:`JsonRequest` class adds two new argument to the constructor. The
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -556,7 +556,7 @@ dealing with JSON requests.
|
|||
|
||||
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
|
||||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
|
|
@ -581,7 +581,7 @@ Sending a JSON POST request with a JSON payload::
|
|||
Response objects
|
||||
================
|
||||
|
||||
.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
|
||||
.. autoclass:: Response
|
||||
|
||||
A :class:`Response` object represents an HTTP response, which is usually
|
||||
downloaded (by the Downloader) and fed to the Spiders for processing.
|
||||
|
|
@ -596,8 +596,8 @@ Response objects
|
|||
(for single valued headers) or lists (for multi-valued headers).
|
||||
:type headers: dict
|
||||
|
||||
:param body: the response body. To access the decoded text as str (unicode
|
||||
in Python 2) you can use ``response.text`` from an encoding-aware
|
||||
:param body: the response body. To access the decoded text as str you can use
|
||||
``response.text`` from an encoding-aware
|
||||
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
|
||||
such as :class:`TextResponse`.
|
||||
:type body: bytes
|
||||
|
|
@ -721,7 +721,7 @@ TextResponse objects
|
|||
:class:`Response` class, which is meant to be used only for binary data,
|
||||
such as images, sounds or any media file.
|
||||
|
||||
:class:`TextResponse` objects support a new constructor argument, in
|
||||
:class:`TextResponse` objects support a new ``__init__`` method argument, in
|
||||
addition to the base :class:`Response` objects. The remaining functionality
|
||||
is the same as for the :class:`Response` class and is not documented here.
|
||||
|
||||
|
|
@ -755,7 +755,7 @@ TextResponse objects
|
|||
A string with the encoding of this response. The encoding is resolved by
|
||||
trying the following mechanisms, in order:
|
||||
|
||||
1. the encoding passed in the constructor ``encoding`` argument
|
||||
1. the encoding passed in the ``__init__`` method ``encoding`` argument
|
||||
|
||||
2. the encoding declared in the Content-Type HTTP header. If this
|
||||
encoding is not valid (ie. unknown), it is ignored and the next
|
||||
|
|
@ -816,5 +816,4 @@ XmlResponse objects
|
|||
adds encoding auto-discovering support by looking into the XML declaration
|
||||
line. See :attr:`TextResponse.encoding`.
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
|
||||
|
|
|
|||
|
|
@ -188,7 +188,6 @@ AWS_ENDPOINT_URL
|
|||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
|
|
@ -199,7 +198,6 @@ Default: ``None``
|
|||
|
||||
Use this option if you want to disable SSL connection for communication with
|
||||
S3 or S3-like storage. By default SSL will be used.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: AWS_VERIFY
|
||||
|
||||
|
|
@ -209,7 +207,7 @@ AWS_VERIFY
|
|||
Default: ``None``
|
||||
|
||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur. Only supported with ``botocore`` library.
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
|
|
@ -219,7 +217,6 @@ AWS_REGION_NAME
|
|||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
|
|
@ -229,8 +226,7 @@ BOT_NAME
|
|||
Default: ``'scrapybot'``
|
||||
|
||||
The name of the bot implemented by this Scrapy project (also known as the
|
||||
project name). This will be used to construct the User-Agent by default, and
|
||||
also for logging.
|
||||
project name). This name will be used for the logging too.
|
||||
|
||||
It's automatically populated with your project name when you create your
|
||||
project with the :command:`startproject` command.
|
||||
|
|
@ -796,6 +792,7 @@ Default: ``True``
|
|||
|
||||
Whether or not to use passive mode when initiating FTP transfers.
|
||||
|
||||
.. reqmeta:: ftp_password
|
||||
.. setting:: FTP_PASSWORD
|
||||
|
||||
FTP_PASSWORD
|
||||
|
|
@ -814,6 +811,7 @@ in ``Request`` meta.
|
|||
|
||||
.. _RFC 1635: https://tools.ietf.org/html/rfc1635
|
||||
|
||||
.. reqmeta:: ftp_user
|
||||
.. setting:: FTP_USER
|
||||
|
||||
FTP_USER
|
||||
|
|
@ -951,7 +949,7 @@ LOGSTATS_INTERVAL
|
|||
|
||||
Default: ``60.0``
|
||||
|
||||
The interval (in seconds) between each logging printout of the stats
|
||||
The interval (in seconds) between each logging printout of the stats
|
||||
by :class:`~scrapy.extensions.logstats.LogStats`.
|
||||
|
||||
.. setting:: MEMDEBUG_ENABLED
|
||||
|
|
@ -1165,7 +1163,7 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
|
|||
ROBOTSTXT_PARSER
|
||||
----------------
|
||||
|
||||
Default: ``'scrapy.robotstxt.PythonRobotParser'``
|
||||
Default: ``'scrapy.robotstxt.ProtegoRobotParser'``
|
||||
|
||||
The parser backend to use for parsing ``robots.txt`` files. For more information see
|
||||
:ref:`topics-dlmw-robots`.
|
||||
|
|
|
|||
|
|
@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
|
|||
Deferred signal handlers
|
||||
========================
|
||||
|
||||
Some signals support returning `Twisted deferreds`_ from their handlers, see
|
||||
the :ref:`topics-signals-ref` below to know which ones.
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, see the :ref:`topics-signals-ref` below to know
|
||||
which ones.
|
||||
|
||||
.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
@ -155,8 +155,8 @@ item_error
|
|||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
spider_closed
|
||||
-------------
|
||||
|
|
@ -236,8 +236,8 @@ spider_error
|
|||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
:param response: the response being processed when the exception was raised
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -333,5 +333,3 @@ response_downloaded
|
|||
|
||||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
|
|
|
|||
|
|
@ -72,8 +72,6 @@ scrapy.Spider
|
|||
spider that crawls ``mywebsite.com`` would often be called
|
||||
``mywebsite``.
|
||||
|
||||
.. note:: In Python 2 this must be ASCII only.
|
||||
|
||||
.. attribute:: allowed_domains
|
||||
|
||||
An optional list of strings containing domains that this spider is
|
||||
|
|
@ -692,7 +690,7 @@ SitemapSpider
|
|||
|
||||
.. method:: sitemap_filter(entries)
|
||||
|
||||
This is a filter funtion that could be overridden to select sitemap entries
|
||||
This is a filter function that could be overridden to select sitemap entries
|
||||
based on their attributes.
|
||||
|
||||
For example::
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
from __future__ import print_function
|
||||
from time import time
|
||||
from collections import deque
|
||||
from twisted.web.server import Site, NOT_DONE_YET
|
||||
|
|
|
|||
|
|
@ -1,25 +1,210 @@
|
|||
#compdef scrapy
|
||||
|
||||
# zsh completion for the Scrapy command-line tool
|
||||
|
||||
_scrapy() {
|
||||
local curcontext="$curcontext" cmd spiders
|
||||
local context state state_descr line
|
||||
typeset -A opt_args
|
||||
cmd=$words[2]
|
||||
|
||||
case "$cmd" in
|
||||
crawl|edit|check)
|
||||
spiders=$(scrapy list 2>/dev/null) || spiders=""
|
||||
if [[ -n "$spiders" ]]; then
|
||||
compadd `echo $spiders`
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if [[ CURRENT -eq 2 ]]; then
|
||||
_arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
|
||||
fi
|
||||
;;
|
||||
_arguments \
|
||||
"(- 1 *)--help[Help]" \
|
||||
"1: :->command" \
|
||||
"*:: :->args"
|
||||
|
||||
case $state in
|
||||
command)
|
||||
_scrapy_cmds
|
||||
;;
|
||||
args)
|
||||
case $words[1] in
|
||||
bench)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
fetch)
|
||||
local options=(
|
||||
'--headers[print response HTTP headers instead of body]'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'1::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
genspider)
|
||||
local options=(
|
||||
{-l,--list}'[List available templates]'
|
||||
{-e,--edit}'[Edit spider after creating it]'
|
||||
'--force[If the spider already exists, overwrite it with the template]'
|
||||
{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
|
||||
{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
|
||||
'1:name:(NAME)'
|
||||
'2:domain:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
runspider)
|
||||
local options=(
|
||||
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider file:_files -g \*.py'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
settings)
|
||||
local options=(
|
||||
'--get=[print raw setting value]:option:(SETTING)'
|
||||
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
|
||||
'--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
|
||||
'--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
|
||||
'--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
shell)
|
||||
local options=(
|
||||
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'::file:_files -g \*.html'
|
||||
'::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
startproject)
|
||||
local options=(
|
||||
'1:name:(NAME)'
|
||||
'2:dir:_dir_list'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
version)
|
||||
local options=(
|
||||
{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
view)
|
||||
local options=(
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
check)
|
||||
local options=(
|
||||
'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
|
||||
{-v,--verbose}'[print contract tests for all spiders]'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
crawl)
|
||||
local options=(
|
||||
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
edit)
|
||||
local options=(
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
list)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
parse)
|
||||
local options=(
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
|
||||
'--pipelines[process items through pipelines]'
|
||||
"--nolinks[don't show links to follow (extracted requests)]"
|
||||
"--noitems[don't show scraped items]"
|
||||
'--nocolour[avoid using pygments to colorize the output]'
|
||||
{-r,--rules}'[use CrawlSpider rules to discover the callback]'
|
||||
{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
|
||||
{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
|
||||
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
|
||||
{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
|
||||
{-v,--verbose}'[print each depth level one by one]'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
_scrapy
|
||||
_scrapy_cmds() {
|
||||
local -a commands project_commands
|
||||
commands=(
|
||||
'bench:Run quick benchmark test'
|
||||
'fetch:Fetch a URL using the Scrapy downloader'
|
||||
'genspider:Generate new spider using pre-defined templates'
|
||||
'runspider:Run a self-contained spider (without creating a project)'
|
||||
'settings:Get settings values'
|
||||
'shell:Interactive scraping console'
|
||||
'startproject:Create new project'
|
||||
'version:Print Scrapy version'
|
||||
'view:Open URL in browser, as seen by Scrapy'
|
||||
)
|
||||
project_commands=(
|
||||
'check:Check spider contracts'
|
||||
'crawl:Run a spider'
|
||||
'edit:Edit spider'
|
||||
'list:List available spiders'
|
||||
'parse:Parse URL (using its spider) and print the results'
|
||||
)
|
||||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
commands=(${commands[@]} ${project_commands[@]})
|
||||
fi
|
||||
_describe -t common-commands 'common commands' commands
|
||||
}
|
||||
|
||||
_scrapy_glb_opts() {
|
||||
local -a options
|
||||
options=(
|
||||
'(- *)'{-h,--help}'[show this help message and exit]'
|
||||
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
|
||||
'--pidfile=[write process ID to FILE]:file:_files'
|
||||
'--profile=[write python cProfile stats to FILE]:file:_files'
|
||||
'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
|
||||
'(-L --loglevel --logfile)--nolog[disable logging completely]'
|
||||
'--pdb[enable pdb on failure]'
|
||||
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
)
|
||||
options=(${options[@]} "$@")
|
||||
_arguments $options
|
||||
}
|
||||
|
||||
_httpie_urls() {
|
||||
|
||||
local ret=1
|
||||
|
||||
if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
|
||||
local expl
|
||||
compset -S '[^:/]*' && compstate[to_end]=''
|
||||
_wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
|
||||
else
|
||||
_urls && ret=0
|
||||
fi
|
||||
|
||||
return $ret
|
||||
|
||||
}
|
||||
|
||||
_scrapy_spiders() {
|
||||
|
||||
local ret=1
|
||||
|
||||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
compadd -S '' $(scrapy list) && ret=0
|
||||
else
|
||||
compadd -S '' SPIDER && ret=0
|
||||
fi
|
||||
|
||||
return $ret
|
||||
}
|
||||
|
||||
_scrapy $@
|
||||
|
|
|
|||
246
pytest.ini
246
pytest.ini
|
|
@ -2,5 +2,249 @@
|
|||
usefixtures = chdir
|
||||
python_files=test_*.py __init__.py
|
||||
python_classes=
|
||||
addopts = --doctest-modules --assert=plain
|
||||
addopts =
|
||||
--assert=plain
|
||||
--doctest-modules
|
||||
--ignore=docs/_ext
|
||||
--ignore=docs/conf.py
|
||||
--ignore=docs/news.rst
|
||||
--ignore=docs/topics/developer-tools.rst
|
||||
--ignore=docs/topics/dynamic-content.rst
|
||||
--ignore=docs/topics/items.rst
|
||||
--ignore=docs/topics/leaks.rst
|
||||
--ignore=docs/topics/loaders.rst
|
||||
--ignore=docs/topics/selectors.rst
|
||||
--ignore=docs/topics/shell.rst
|
||||
--ignore=docs/topics/stats.rst
|
||||
--ignore=docs/topics/telnetconsole.rst
|
||||
--ignore=docs/utils
|
||||
twisted = 1
|
||||
flake8-ignore =
|
||||
# Files that are only meant to provide top-level imports are expected not
|
||||
# to use any of their imports:
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
# Issues pending a review:
|
||||
# extras
|
||||
extras/qps-bench-server.py E261 E501
|
||||
extras/qpsclient.py E501 E261 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128 E502 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/startproject.py E502 E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E502 E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E261 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E502 E126 E226
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E502 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E502 E128 E123
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128 W293 W291
|
||||
scrapy/http/response/text.py E501 W293 E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E502 E128
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
scrapy/pipelines/images.py E265 E501
|
||||
scrapy/pipelines/media.py E125 E501 E266
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501 E261
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/conf.py E402 E502 E501
|
||||
scrapy/utils/console.py E261 E306 E305
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/engine.py E261
|
||||
scrapy/utils/gz.py E305 E501 W504
|
||||
scrapy/utils/http.py F403 E226
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501 E701
|
||||
scrapy/utils/log.py E128 W503
|
||||
scrapy/utils/markup.py F403 W292
|
||||
scrapy/utils/misc.py E501 E226
|
||||
scrapy/utils/multipart.py F403 W292
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E226
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E271 E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/test.py E501
|
||||
scrapy/utils/url.py E501 F403 E128 F405
|
||||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/_monkeypatches.py W293
|
||||
scrapy/cmdline.py E502 E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E501 E202
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501 E261 E226
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501 W293
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/middleware.py E502 E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/robotstxt.py E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py E225 F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/pipelines.py F841 E226
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501 E261
|
||||
tests/test_command_parse.py E501 E128 E303 E226
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128 W293
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501 E305
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E502 E128 E261
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_response.py E501 E301 E502 E128 E265
|
||||
tests/test_item.py E701 E128 F841 E306
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501 E305
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E305
|
||||
tests/test_robotstxt_interface.py E501 W291 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501 E226
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py E501 W293 E226
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E701 E741
|
||||
tests/test_utils_conf.py E501 E303 E128
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E261 E501 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_http.py E501 E502 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128 E305
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E731 E226
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/test_cmdline/__init__.py E502 E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
parsel>=1.5.0
|
||||
PyDispatcher>=2.0.5
|
||||
w3lib>=1.17.0
|
||||
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
|
||||
# Reference versions taken from
|
||||
# https://packages.ubuntu.com/xenial/python/
|
||||
# https://packages.ubuntu.com/xenial/zope/
|
||||
cssselect>=0.9.1
|
||||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
Twisted>=16.0.0
|
||||
zope.interface>=4.1.3
|
||||
|
|
@ -13,5 +13,4 @@ cryptography>=2.0 # Earlier versions would fail to install
|
|||
cssselect>=0.9.1
|
||||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
zope.interface>=4.1.3
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
1.7.0
|
||||
1.8.0
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ del pkgutil
|
|||
|
||||
# Check minimum required Python version
|
||||
import sys
|
||||
if sys.version_info < (2, 7):
|
||||
print("Scrapy %s requires Python 2.7" % __version__)
|
||||
if sys.version_info < (3, 5):
|
||||
print("Scrapy %s requires Python 3.5" % __version__)
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore noisy twisted deprecation warnings
|
||||
|
|
@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
|
|||
del warnings
|
||||
|
||||
# Apply monkey patches to fix issues in external libraries
|
||||
from . import _monkeypatches
|
||||
from scrapy import _monkeypatches
|
||||
del _monkeypatches
|
||||
|
||||
from twisted import version as _txv
|
||||
|
|
|
|||
|
|
@ -1,14 +1,4 @@
|
|||
import six
|
||||
from six.moves import copyreg
|
||||
|
||||
|
||||
if six.PY2:
|
||||
from urlparse import urlparse
|
||||
|
||||
# workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
|
||||
if urlparse('s3://bucket/key?key=value').query != 'key=value':
|
||||
from urlparse import uses_query
|
||||
uses_query.append('s3')
|
||||
import copyreg
|
||||
|
||||
|
||||
# Undo what Twisted's perspective broker adds to pickle register
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class ScrapyCommand(object):
|
|||
|
||||
def help(self):
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since not post-formatting will
|
||||
"help" command. It can contain newlines, since no post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
|
||||
from six.moves.urllib.parse import urlencode
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from __future__ import print_function
|
||||
import time
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from unittest import TextTestRunner, TextTestResult as _TextTestResult
|
||||
|
||||
|
|
@ -96,4 +94,3 @@ class Command(ScrapyCommand):
|
|||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from __future__ import print_function
|
||||
import sys, six
|
||||
import sys
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -8,6 +7,7 @@ from scrapy.exceptions import UsageError
|
|||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
@ -45,8 +45,7 @@ class Command(ScrapyCommand):
|
|||
self._print_bytes(response.body)
|
||||
|
||||
def _print_bytes(self, bytes_):
|
||||
bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
|
||||
bytes_writer.write(bytes_ + b'\n')
|
||||
sys.stdout.buffer.write(bytes_ + b'\n')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
import shutil
|
||||
import string
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import print_function
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import json
|
||||
import logging
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from __future__ import print_function
|
||||
import json
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import re
|
||||
import os
|
||||
import string
|
||||
|
|
@ -119,4 +118,3 @@ class Command(ScrapyCommand):
|
|||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'project')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from __future__ import print_function
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.utils.versions import scrapy_components_versions
|
||||
|
|
@ -30,4 +28,3 @@ class Command(ScrapyCommand):
|
|||
print(patt % (name, version))
|
||||
else:
|
||||
print("Scrapy %s" % scrapy.__version__)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from scrapy.commands import fetch, ScrapyCommand
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
||||
class Command(fetch.Command):
|
||||
|
||||
def short_desc(self):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from scrapy.item import BaseItem
|
|||
from scrapy.http import Request
|
||||
from scrapy.exceptions import ContractFail
|
||||
|
||||
from . import Contract
|
||||
from scrapy.contracts import Contract
|
||||
|
||||
|
||||
# contracts
|
||||
|
|
|
|||
|
|
@ -1,19 +1,16 @@
|
|||
from __future__ import absolute_import
|
||||
import random
|
||||
import warnings
|
||||
from time import time
|
||||
from datetime import datetime
|
||||
from collections import deque
|
||||
|
||||
import six
|
||||
from twisted.internet import reactor, defer, task
|
||||
|
||||
from scrapy.utils.defer import mustbe_deferred
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.resolver import dnscache
|
||||
from scrapy import signals
|
||||
from .middleware import DownloaderMiddlewareManager
|
||||
from .handlers import DownloadHandlers
|
||||
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
|
||||
|
||||
class Slot(object):
|
||||
|
|
@ -190,7 +187,7 @@ class Downloader(object):
|
|||
|
||||
def close(self):
|
||||
self._slot_gc_loop.stop()
|
||||
for slot in six.itervalues(self.slots):
|
||||
for slot in self.slots.values():
|
||||
slot.close()
|
||||
|
||||
def _slot_gc(self, age=60):
|
||||
|
|
|
|||
|
|
@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
"""
|
||||
Twisted-recommended context factory for web clients.
|
||||
|
||||
Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
|
||||
|
||||
creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
|
||||
except this context factory allows setting the TLS/SSL method to use.
|
||||
The default is to use a
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
|
||||
have special requirements you can leave this as-is.
|
||||
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation.
|
||||
:meth:`creatorForNetloc` is the same as
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
|
||||
factory allows setting the TLS/SSL method to use.
|
||||
|
||||
The default OpenSSL method is ``TLS_METHOD`` (also called
|
||||
``SSLv23_METHOD``) which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
"""Download handlers for different schemes"""
|
||||
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
import six
|
||||
|
||||
from scrapy.exceptions import NotSupported, NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
|
|
@ -22,7 +23,7 @@ class DownloadHandlers(object):
|
|||
self._notconfigured = {} # remembers failed handlers
|
||||
handlers = without_none_values(
|
||||
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
|
||||
for scheme, clspath in six.iteritems(handlers):
|
||||
for scheme, clspath in handlers.items():
|
||||
self._schemes[scheme] = clspath
|
||||
self._load_handler(scheme, skip_lazy=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
|
|||
respcls = responsetypes.from_mimetype(uri.media_type)
|
||||
|
||||
resp_kwargs = {}
|
||||
if (issubclass(respcls, TextResponse) and
|
||||
uri.media_type.split('/')[0] == 'text'):
|
||||
if (issubclass(respcls, TextResponse)
|
||||
and uri.media_type.split('/')[0] == 'text'):
|
||||
charset = uri.media_type_parameters.get('charset')
|
||||
resp_kwargs['encoding'] = charset
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
|
|||
|
||||
import re
|
||||
from io import BytesIO
|
||||
from six.moves.urllib.parse import unquote
|
||||
from urllib.parse import unquote
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.protocols.ftp import FTPClient, CommandFailed
|
||||
|
|
@ -112,4 +112,3 @@ class FTPDownloadHandler(object):
|
|||
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
|
||||
return Response(url=request.url, status=httpcode, body=to_bytes(message))
|
||||
raise result.type(result.value)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from __future__ import absolute_import
|
||||
from .http10 import HTTP10DownloadHandler
|
||||
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http11 import (
|
||||
HTTP11DownloadHandler as HTTPDownloadHandler,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
|
||||
import re
|
||||
import logging
|
||||
import warnings
|
||||
from io import BytesIO
|
||||
from time import time
|
||||
import warnings
|
||||
from six.moves.urllib.parse import urldefrag
|
||||
from urllib.parse import urldefrag
|
||||
|
||||
from zope.interface import implementer
|
||||
from twisted.internet import defer, reactor, protocol
|
||||
|
|
@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
|
|||
r"""
|
||||
Return binary content of a CONNECT request.
|
||||
|
||||
>>> from scrapy.utils.python import to_native_str as s
|
||||
>>> from scrapy.utils.python import to_unicode as s
|
||||
>>> s(tunnel_request_data("example.com", 8080))
|
||||
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
|
||||
>>> s(tunnel_request_data("example.com", 8080, b"123"))
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from six.moves.urllib.parse import unquote
|
||||
from urllib.parse import unquote
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.boto import is_botocore
|
||||
from .http import HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
||||
|
||||
|
||||
def _get_boto_connection():
|
||||
|
|
@ -21,7 +21,7 @@ def _get_boto_connection():
|
|||
return http_request.headers
|
||||
|
||||
try:
|
||||
import boto.auth
|
||||
import boto.auth # noqa: F401
|
||||
except ImportError:
|
||||
_S3Connection = _v19_S3Connection
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
import six
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
|
|
@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
response = yield method(request=request, spider=spider)
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
|
||||
(six.get_method_self(method).__class__.__name__, response.__class__.__name__))
|
||||
(method.__self__.__class__.__name__, response.__class__.__name__))
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue((yield download_func(request=request, spider=spider)))
|
||||
|
|
@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
response = yield method(request=request, response=response, spider=spider)
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
|
||||
(six.get_method_self(method).__class__.__name__, type(response)))
|
||||
(method.__self__.__class__.__name__, type(response)))
|
||||
if isinstance(response, Request):
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(response)
|
||||
|
|
@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
response = yield method(request=request, exception=exception, spider=spider)
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
|
||||
(six.get_method_self(method).__class__.__name__, type(response)))
|
||||
(method.__self__.__class__.__name__, type(response)))
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(_failure)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from time import time
|
||||
from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
|
||||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.client import HTTPClientFactory
|
||||
from twisted.web.http import HTTPClient
|
||||
|
|
@ -157,4 +157,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
def gotHeaders(self, headers):
|
||||
self.headers_time = time()
|
||||
self.response_headers = headers
|
||||
|
||||
|
|
|
|||
|
|
@ -231,9 +231,9 @@ class Scraper(object):
|
|||
signal=signals.item_dropped, item=item, response=response,
|
||||
spider=spider, exception=output.value)
|
||||
else:
|
||||
logger.error('Error processing %(item)s', {'item': item},
|
||||
exc_info=failure_to_exc_info(output),
|
||||
extra={'spider': spider})
|
||||
logkws = self.logformatter.error(item, ex, response, spider)
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
|
||||
exc_info=failure_to_exc_info(output))
|
||||
return self.signals.send_catch_log_deferred(
|
||||
signal=signals.item_error, item=item, response=response,
|
||||
spider=spider, failure=output)
|
||||
|
|
@ -244,4 +244,3 @@ class Scraper(object):
|
|||
return self.signals.send_catch_log_deferred(
|
||||
signal=signals.item_scraped, item=output, response=response,
|
||||
spider=spider)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
|
|||
"""
|
||||
from itertools import chain, islice
|
||||
|
||||
import six
|
||||
from twisted.python.failure import Failure
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
|
|
@ -36,9 +35,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
|
||||
|
||||
def scrape_response(self, scrape_func, response, request, spider):
|
||||
fname = lambda f:'%s.%s' % (
|
||||
six.get_method_self(f).__class__.__name__,
|
||||
six.get_method_function(f).__name__)
|
||||
fname = lambda f: '%s.%s' % (
|
||||
f.__self__.__class__.__name__,
|
||||
f.__func__.__name__)
|
||||
|
||||
def process_spider_input(response):
|
||||
for method in self.methods['process_spider_input']:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import six
|
||||
import signal
|
||||
import logging
|
||||
import pprint
|
||||
import signal
|
||||
import warnings
|
||||
|
||||
import sys
|
||||
from twisted.internet import reactor, defer
|
||||
from zope.interface.verify import verifyClass, DoesNotImplement
|
||||
|
||||
|
|
@ -22,6 +21,7 @@ from scrapy.utils.log import (
|
|||
get_scrapy_root_handler, install_scrapy_root_handler)
|
||||
from scrapy import signals
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -46,7 +46,8 @@ class Crawler(object):
|
|||
logging.root.addHandler(handler)
|
||||
|
||||
d = dict(overridden_settings(self.settings))
|
||||
logger.info("Overridden settings: %(settings)r", {'settings': d})
|
||||
logger.info("Overridden settings:\n%(settings)s",
|
||||
{'settings': pprint.pformat(d)})
|
||||
|
||||
if get_scrapy_root_handler() is not None:
|
||||
# scrapy root handler already installed: update it with new settings
|
||||
|
|
@ -88,20 +89,9 @@ class Crawler(object):
|
|||
yield self.engine.open_spider(self.spider, start_requests)
|
||||
yield defer.maybeDeferred(self.engine.start)
|
||||
except Exception:
|
||||
# In Python 2 reraising an exception after yield discards
|
||||
# the original traceback (see https://bugs.python.org/issue7563),
|
||||
# so sys.exc_info() workaround is used.
|
||||
# This workaround also works in Python 3, but it is not needed,
|
||||
# and it is slower, so in Python 3 we use native `raise`.
|
||||
if six.PY2:
|
||||
exc_info = sys.exc_info()
|
||||
|
||||
self.crawling = False
|
||||
if self.engine is not None:
|
||||
yield self.engine.close()
|
||||
|
||||
if six.PY2:
|
||||
six.reraise(*exc_info)
|
||||
raise
|
||||
|
||||
def _create_spider(self, *args, **kwargs):
|
||||
|
|
@ -122,7 +112,7 @@ class Crawler(object):
|
|||
class CrawlerRunner(object):
|
||||
"""
|
||||
This is a convenient helper class that keeps track of, manages and runs
|
||||
crawlers inside an already setup Twisted `reactor`_.
|
||||
crawlers inside an already setup :mod:`~twisted.internet.reactor`.
|
||||
|
||||
The CrawlerRunner object must be instantiated with a
|
||||
:class:`~scrapy.settings.Settings` object.
|
||||
|
|
@ -216,7 +206,7 @@ class CrawlerRunner(object):
|
|||
return self._create_crawler(crawler_or_spidercls)
|
||||
|
||||
def _create_crawler(self, spidercls):
|
||||
if isinstance(spidercls, six.string_types):
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings)
|
||||
|
||||
|
|
@ -245,12 +235,13 @@ class CrawlerProcess(CrawlerRunner):
|
|||
A class to run multiple scrapy crawlers in a process simultaneously.
|
||||
|
||||
This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
|
||||
for starting a Twisted `reactor`_ and handling shutdown signals, like the
|
||||
keyboard interrupt command Ctrl-C. It also configures top-level logging.
|
||||
for starting a :mod:`~twisted.internet.reactor` and handling shutdown
|
||||
signals, like the keyboard interrupt command Ctrl-C. It also configures
|
||||
top-level logging.
|
||||
|
||||
This utility should be a better fit than
|
||||
:class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
|
||||
Twisted `reactor`_ within your application.
|
||||
:mod:`~twisted.internet.reactor` within your application.
|
||||
|
||||
The CrawlerProcess object must be instantiated with a
|
||||
:class:`~scrapy.settings.Settings` object.
|
||||
|
|
@ -285,9 +276,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def start(self, stop_after_crawl=True):
|
||||
"""
|
||||
This method starts a Twisted `reactor`_, adjusts its pool size to
|
||||
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
|
||||
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import absolute_import
|
||||
import re
|
||||
import logging
|
||||
|
||||
import six
|
||||
from w3lib import html
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -67,7 +65,9 @@ class AjaxCrawlMiddleware(object):
|
|||
|
||||
|
||||
# XXX: move it to w3lib?
|
||||
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
|
||||
_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
|
||||
|
||||
|
||||
def _has_ajaxcrawlable_meta(text):
|
||||
"""
|
||||
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import os
|
||||
import six
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -53,7 +52,7 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _debug_cookie(self, request, spider):
|
||||
if self.debug:
|
||||
cl = [to_native_str(c, errors='replace')
|
||||
cl = [to_unicode(c, errors='replace')
|
||||
for c in request.headers.getlist('Cookie')]
|
||||
if cl:
|
||||
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
|
||||
|
|
@ -62,7 +61,7 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _debug_set_cookie(self, response, spider):
|
||||
if self.debug:
|
||||
cl = [to_native_str(c, errors='replace')
|
||||
cl = [to_unicode(c, errors='replace')
|
||||
for c in response.headers.getlist('Set-Cookie')]
|
||||
if cl:
|
||||
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
|
||||
|
|
@ -82,8 +81,10 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _get_request_cookies(self, jar, request):
|
||||
if isinstance(request.cookies, dict):
|
||||
cookie_list = [{'name': k, 'value': v} for k, v in \
|
||||
six.iteritems(request.cookies)]
|
||||
cookie_list = [
|
||||
{'name': k, 'value': v}
|
||||
for k, v in request.cookies.items()
|
||||
]
|
||||
else:
|
||||
cookie_list = request.cookies
|
||||
|
||||
|
|
|
|||
|
|
@ -4,20 +4,15 @@ and extract the potentially compressed responses that may arrive.
|
|||
|
||||
import bz2
|
||||
import gzip
|
||||
import zipfile
|
||||
import tarfile
|
||||
import logging
|
||||
import tarfile
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
from tempfile import mktemp
|
||||
|
||||
import six
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -79,7 +74,7 @@ class DecompressionMiddleware(object):
|
|||
if not response.body:
|
||||
return response
|
||||
|
||||
for fmt, func in six.iteritems(self._formats):
|
||||
for fmt, func in self._formats.items():
|
||||
new_response = func(response)
|
||||
if new_response:
|
||||
logger.debug('Decompressed response with format: %(responsefmt)s',
|
||||
|
|
|
|||
|
|
@ -1,11 +1,19 @@
|
|||
from email.utils import formatdate
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import base64
|
||||
from six.moves.urllib.parse import unquote, urlunparse
|
||||
from six.moves.urllib.request import getproxies, proxy_bypass
|
||||
try:
|
||||
from urllib2 import _parse_proxy
|
||||
except ImportError:
|
||||
from urllib.request import _parse_proxy
|
||||
from urllib.parse import unquote, urlunparse
|
||||
from urllib.request import getproxies, proxy_bypass, _parse_proxy
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
|
|
@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
|
|||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -70,7 +71,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
if 'Location' not in response.headers or response.status not in allowed_status:
|
||||
return response
|
||||
|
||||
location = safe_url_string(response.headers['location'])
|
||||
location = safe_url_string(response.headers['Location'])
|
||||
if response.headers['Location'].startswith(b'//'):
|
||||
request_scheme = urlparse(request.url).scheme
|
||||
location = request_scheme + '://' + location.lstrip('/')
|
||||
|
||||
redirected_url = urljoin(request.url, location)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
|
|||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import re
|
||||
|
||||
from twisted.internet.defer import Deferred, maybeDeferred
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
import logging
|
||||
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
|
||||
|
||||
class BaseDupeFilter(object):
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ new exceptions here without documenting them there.
|
|||
|
||||
# Internal
|
||||
|
||||
|
||||
class NotConfigured(Exception):
|
||||
"""Indicates a missing configuration situation"""
|
||||
pass
|
||||
|
||||
|
||||
class _InvalidOutput(TypeError):
|
||||
"""
|
||||
Indicates an invalid value has been returned by a middleware's processing method.
|
||||
|
|
@ -18,15 +20,19 @@ class _InvalidOutput(TypeError):
|
|||
"""
|
||||
pass
|
||||
|
||||
|
||||
# HTTP and crawling
|
||||
|
||||
|
||||
class IgnoreRequest(Exception):
|
||||
"""Indicates a decision was made not to process a request"""
|
||||
|
||||
|
||||
class DontCloseSpider(Exception):
|
||||
"""Request the spider not to be closed yet"""
|
||||
pass
|
||||
|
||||
|
||||
class CloseSpider(Exception):
|
||||
"""Raise this from callbacks to request the spider to be closed"""
|
||||
|
||||
|
|
@ -34,30 +40,37 @@ class CloseSpider(Exception):
|
|||
super(CloseSpider, self).__init__()
|
||||
self.reason = reason
|
||||
|
||||
|
||||
# Items
|
||||
|
||||
|
||||
class DropItem(Exception):
|
||||
"""Drop item from the item pipeline"""
|
||||
pass
|
||||
|
||||
|
||||
class NotSupported(Exception):
|
||||
"""Indicates a feature or method is not supported"""
|
||||
pass
|
||||
|
||||
|
||||
# Commands
|
||||
|
||||
|
||||
class UsageError(Exception):
|
||||
"""To indicate a command-line usage error"""
|
||||
def __init__(self, *a, **kw):
|
||||
self.print_help = kw.pop('print_help', True)
|
||||
super(UsageError, self).__init__(*a, **kw)
|
||||
|
||||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
"""Warning category for deprecated features, since the default
|
||||
DeprecationWarning is silenced on Python 2.7+
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class ContractFail(AssertionError):
|
||||
"""Error raised in case of a failing contract"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -4,18 +4,16 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
import pprint
|
||||
import marshal
|
||||
import six
|
||||
from six.moves import cPickle as pickle
|
||||
import warnings
|
||||
import pickle
|
||||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
|
||||
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
import warnings
|
||||
|
||||
|
||||
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
|
||||
|
|
@ -31,7 +29,7 @@ class BaseItemExporter(object):
|
|||
def _configure(self, options, dont_fail=False):
|
||||
"""Configure the exporter by poping options from the ``options`` dict.
|
||||
If dont_fail is set, it won't raise an exception on unexpected options
|
||||
(useful for using with keyword arguments in subclasses constructors)
|
||||
(useful for using with keyword arguments in subclasses ``__init__`` methods)
|
||||
"""
|
||||
self.encoding = options.pop('encoding', None)
|
||||
self.fields_to_export = options.pop('fields_to_export', None)
|
||||
|
|
@ -61,9 +59,9 @@ class BaseItemExporter(object):
|
|||
include_empty = self.export_empty_fields
|
||||
if self.fields_to_export is None:
|
||||
if include_empty and not isinstance(item, dict):
|
||||
field_iter = six.iterkeys(item.fields)
|
||||
field_iter = item.fields.keys()
|
||||
else:
|
||||
field_iter = six.iterkeys(item)
|
||||
field_iter = item.keys()
|
||||
else:
|
||||
if include_empty:
|
||||
field_iter = self.fields_to_export
|
||||
|
|
@ -143,11 +141,11 @@ class XmlItemExporter(BaseItemExporter):
|
|||
|
||||
def _beautify_newline(self, new_item=False):
|
||||
if self.indent is not None and (self.indent > 0 or new_item):
|
||||
self._xg_characters('\n')
|
||||
self.xg.characters('\n')
|
||||
|
||||
def _beautify_indent(self, depth=1):
|
||||
if self.indent:
|
||||
self._xg_characters(' ' * self.indent * depth)
|
||||
self.xg.characters(' ' * self.indent * depth)
|
||||
|
||||
def start_exporting(self):
|
||||
self.xg.startDocument()
|
||||
|
|
@ -181,27 +179,13 @@ class XmlItemExporter(BaseItemExporter):
|
|||
for value in serialized_value:
|
||||
self._export_xml_field('value', value, depth=depth+1)
|
||||
self._beautify_indent(depth=depth)
|
||||
elif isinstance(serialized_value, six.text_type):
|
||||
self._xg_characters(serialized_value)
|
||||
elif isinstance(serialized_value, str):
|
||||
self.xg.characters(serialized_value)
|
||||
else:
|
||||
self._xg_characters(str(serialized_value))
|
||||
self.xg.characters(str(serialized_value))
|
||||
self.xg.endElement(name)
|
||||
self._beautify_newline()
|
||||
|
||||
# Workaround for https://bugs.python.org/issue17606
|
||||
# Before Python 2.7.4 xml.sax.saxutils required bytes;
|
||||
# since 2.7.4 it requires unicode. The bug is likely to be
|
||||
# fixed in 2.7.6, but 2.7.6 will still support unicode,
|
||||
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
|
||||
if sys.version_info[:3] >= (2, 7, 4):
|
||||
def _xg_characters(self, serialized_value):
|
||||
if not isinstance(serialized_value, six.text_type):
|
||||
serialized_value = serialized_value.decode(self.encoding)
|
||||
return self.xg.characters(serialized_value)
|
||||
else: # pragma: no cover
|
||||
def _xg_characters(self, serialized_value):
|
||||
return self.xg.characters(serialized_value)
|
||||
|
||||
|
||||
class CsvItemExporter(BaseItemExporter):
|
||||
|
||||
|
|
@ -216,7 +200,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
write_through=True,
|
||||
encoding=self.encoding,
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
) if six.PY3 else file
|
||||
)
|
||||
self.csv_writer = csv.writer(self.stream, **kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
|
|
@ -246,7 +230,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def _build_row(self, values):
|
||||
for s in values:
|
||||
try:
|
||||
yield to_native_str(s, self.encoding)
|
||||
yield to_unicode(s, self.encoding)
|
||||
except TypeError:
|
||||
yield s
|
||||
|
||||
|
|
@ -334,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
|
|||
if is_listlike(value):
|
||||
return [self._serialize_value(v) for v in value]
|
||||
encode_func = to_bytes if self.binary else to_unicode
|
||||
if isinstance(value, (six.text_type, bytes)):
|
||||
if isinstance(value, (str, bytes)):
|
||||
return encode_func(value, encoding=self.encoding)
|
||||
return value
|
||||
|
||||
def _serialize_dict(self, value):
|
||||
for key, val in six.iteritems(value):
|
||||
for key, val in value.items():
|
||||
key = to_bytes(key) if self.binary else key
|
||||
yield key, self._serialize_value(val)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
|
|||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.utils.conf import build_component_list
|
||||
|
||||
|
||||
class ExtensionManager(MiddlewareManager):
|
||||
|
||||
component_name = 'extension'
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
"""
|
||||
Extension for collecting core stats like items scraped and start/finish times
|
||||
"""
|
||||
import datetime
|
||||
from datetime import datetime
|
||||
|
||||
from scrapy import signals
|
||||
|
||||
|
||||
class CoreStats(object):
|
||||
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
self.start_time = None
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
|
|
@ -21,11 +23,12 @@ class CoreStats(object):
|
|||
return o
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
|
||||
self.start_time = datetime.utcnow()
|
||||
self.stats.set_value('start_time', self.start_time, spider=spider)
|
||||
|
||||
def spider_closed(self, spider, reason):
|
||||
finish_time = datetime.datetime.utcnow()
|
||||
elapsed_time = finish_time - self.stats.get_value('start_time')
|
||||
finish_time = datetime.utcnow()
|
||||
elapsed_time = finish_time - self.start_time
|
||||
elapsed_time_seconds = elapsed_time.total_seconds()
|
||||
self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
|
||||
self.stats.set_value('finish_time', finish_time, spider=spider)
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ import logging
|
|||
import posixpath
|
||||
from tempfile import NamedTemporaryFile
|
||||
from datetime import datetime
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse, unquote
|
||||
from urllib.parse import urlparse, unquote
|
||||
from ftplib import FTP
|
||||
|
||||
from zope.interface import Interface, implementer
|
||||
|
|
@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
|
|||
|
||||
def __init__(self, uri, _stdout=None):
|
||||
if not _stdout:
|
||||
_stdout = sys.stdout if six.PY2 else sys.stdout.buffer
|
||||
_stdout = sys.stdout.buffer
|
||||
self._stdout = _stdout
|
||||
|
||||
def open(self, spider):
|
||||
|
|
@ -199,9 +198,9 @@ class FeedExporter(object):
|
|||
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.urifmt = settings['FEED_URI']
|
||||
if not self.urifmt:
|
||||
if not settings['FEED_URI']:
|
||||
raise NotConfigured
|
||||
self.urifmt = str(settings['FEED_URI'])
|
||||
self.format = settings['FEED_FORMAT'].lower()
|
||||
self.export_encoding = settings['FEED_EXPORT_ENCODING']
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
|
|
@ -242,7 +241,9 @@ class FeedExporter(object):
|
|||
def close_spider(self, spider):
|
||||
slot = self.slot
|
||||
if not slot.itemcount and not self.store_empty:
|
||||
return
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
if self._exporting:
|
||||
slot.exporter.finish_exporting()
|
||||
self._exporting = False
|
||||
|
|
|
|||
|
|
@ -1,19 +1,20 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
import gzip
|
||||
import logging
|
||||
from six.moves import cPickle as pickle
|
||||
import os
|
||||
import pickle
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from importlib import import_module
|
||||
from time import time
|
||||
from weakref import WeakKeyDictionary
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
|
||||
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
|
||||
|
||||
from scrapy.http import Headers, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -342,75 +343,6 @@ class FilesystemCacheStorage(object):
|
|||
return pickle.load(f)
|
||||
|
||||
|
||||
class LeveldbCacheStorage(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
import leveldb
|
||||
self._leveldb = leveldb
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
self.db = None
|
||||
|
||||
def open_spider(self, spider):
|
||||
dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
|
||||
self.db = self._leveldb.LevelDB(dbpath)
|
||||
|
||||
logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
def close_spider(self, spider):
|
||||
# Do compactation each time to save space and also recreate files to
|
||||
# avoid them being removed in storages with timestamp-based autoremoval.
|
||||
self.db.CompactRange()
|
||||
del self.db
|
||||
garbage_collect()
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
url = data['url']
|
||||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
body = data['body']
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
'headers': dict(response.headers),
|
||||
'body': response.body,
|
||||
}
|
||||
batch = self._leveldb.WriteBatch()
|
||||
batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
|
||||
batch.Put(key + b'_time', to_bytes(str(time())))
|
||||
self.db.Write(batch)
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
try:
|
||||
ts = self.db.Get(key + b'_time')
|
||||
except KeyError:
|
||||
return # not found or invalid entry
|
||||
|
||||
if 0 < self.expiration_secs < time() - float(ts):
|
||||
return # expired
|
||||
|
||||
try:
|
||||
data = self.db.Get(key + b'_data')
|
||||
except KeyError:
|
||||
return # invalid entry
|
||||
else:
|
||||
return pickle.loads(data)
|
||||
|
||||
def _request_key(self, request):
|
||||
return to_bytes(request_fingerprint(request))
|
||||
|
||||
|
||||
|
||||
def parse_cachecontrol(header):
|
||||
"""Parse Cache-Control header
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
|
|||
"""
|
||||
|
||||
import gc
|
||||
import six
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -28,7 +27,7 @@ class MemoryDebugger(object):
|
|||
def spider_closed(self, spider, reason):
|
||||
gc.collect()
|
||||
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
|
||||
for cls, wdict in six.iteritems(live_refs):
|
||||
for cls, wdict in live_refs.items():
|
||||
if not wdict:
|
||||
continue
|
||||
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import os
|
||||
from six.moves import cPickle as pickle
|
||||
import pickle
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.job import job_dir
|
||||
|
||||
|
||||
class SpiderState(object):
|
||||
"""Store and load spider state during a scraping job"""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import time
|
||||
from six.moves.http_cookiejar import (
|
||||
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
|
||||
)
|
||||
from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
|
||||
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
class CookieJar(object):
|
||||
|
|
@ -165,13 +164,13 @@ class WrappedRequest(object):
|
|||
return name in self.request.headers
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return to_native_str(self.request.headers.get(name, default),
|
||||
errors='replace')
|
||||
return to_unicode(self.request.headers.get(name, default),
|
||||
errors='replace')
|
||||
|
||||
def header_items(self):
|
||||
return [
|
||||
(to_native_str(k, errors='replace'),
|
||||
[to_native_str(x, errors='replace') for x in v])
|
||||
(to_unicode(k, errors='replace'),
|
||||
[to_unicode(x, errors='replace') for x in v])
|
||||
for k, v in self.request.headers.items()
|
||||
]
|
||||
|
||||
|
|
@ -189,7 +188,7 @@ class WrappedResponse(object):
|
|||
|
||||
# python3 cookiejars calls get_all
|
||||
def get_all(self, name, default=None):
|
||||
return [to_native_str(v, errors='replace')
|
||||
return [to_unicode(v, errors='replace')
|
||||
for v in self.response.headers.getlist(name)]
|
||||
# python2 cookiejars calls getheaders
|
||||
getheaders = get_all
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import six
|
||||
from w3lib.http import headers_dict_to_raw
|
||||
from scrapy.utils.datatypes import CaselessDict
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -19,7 +18,7 @@ class Headers(CaselessDict):
|
|||
"""Normalize values to bytes"""
|
||||
if value is None:
|
||||
value = []
|
||||
elif isinstance(value, (six.text_type, bytes)):
|
||||
elif isinstance(value, (str, bytes)):
|
||||
value = [value]
|
||||
elif not hasattr(value, '__iter__'):
|
||||
value = [value]
|
||||
|
|
@ -29,10 +28,10 @@ class Headers(CaselessDict):
|
|||
def _tobytes(self, x):
|
||||
if isinstance(x, bytes):
|
||||
return x
|
||||
elif isinstance(x, six.text_type):
|
||||
elif isinstance(x, str):
|
||||
return x.encode(self.encoding)
|
||||
elif isinstance(x, int):
|
||||
return six.text_type(x).encode(self.encoding)
|
||||
return str(x).encode(self.encoding)
|
||||
else:
|
||||
raise TypeError('Unsupported value type: {}'.format(type(x)))
|
||||
|
||||
|
|
@ -68,9 +67,6 @@ class Headers(CaselessDict):
|
|||
self[key] = lst
|
||||
|
||||
def items(self):
|
||||
return list(self.iteritems())
|
||||
|
||||
def iteritems(self):
|
||||
return ((k, self.getlist(k)) for k in self.keys())
|
||||
|
||||
def values(self):
|
||||
|
|
@ -91,5 +87,3 @@ class Headers(CaselessDict):
|
|||
def __copy__(self):
|
||||
return self.__class__(self)
|
||||
copy = __copy__
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ requests in Scrapy.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
import six
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.http.headers import Headers
|
||||
|
|
@ -60,13 +59,13 @@ class Request(object_ref):
|
|||
return self._url
|
||||
|
||||
def _set_url(self, url):
|
||||
if not isinstance(url, six.string_types):
|
||||
if not isinstance(url, str):
|
||||
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
|
||||
|
||||
s = safe_url_string(url, self.encoding)
|
||||
self._url = escape_ajax(s)
|
||||
|
||||
if ':' not in self._url:
|
||||
if ('://' not in self._url) and (not self._url.startswith('data:')):
|
||||
raise ValueError('Missing scheme in request url: %s' % self._url)
|
||||
|
||||
url = property(_get_url, obsolete_setter(_set_url, 'url'))
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
|
|||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin, urlencode
|
||||
from urllib.parse import urljoin, urlencode
|
||||
|
||||
import lxml.html
|
||||
from parsel.selector import create_root_node
|
||||
|
|
@ -104,8 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
|
|||
el = el.getparent()
|
||||
if el is None:
|
||||
break
|
||||
encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
|
||||
raise ValueError('No <form> element found with %s' % encoded)
|
||||
raise ValueError('No <form> element found with %s' % formxpath)
|
||||
|
||||
# If we get here, it means that either formname was None
|
||||
# or invalid
|
||||
|
|
@ -209,7 +207,7 @@ def _get_clickable(clickdata, form):
|
|||
# We didn't find it, so now we build an XPath expression out of the other
|
||||
# arguments, because they can be used as such
|
||||
xpath = u'.//*' + \
|
||||
u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
|
||||
u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
|
||||
el = form.xpath(xpath)
|
||||
if len(el) == 1:
|
||||
return (el[0].get('name'), el[0].get('value') or '')
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
from six.moves import xmlrpc_client as xmlrpclib
|
||||
import xmlrpc.client as xmlrpclib
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ responses in Scrapy.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.headers import Headers
|
||||
|
|
@ -88,7 +88,7 @@ class Response(object_ref):
|
|||
@property
|
||||
def text(self):
|
||||
"""For subclasses of TextResponse, this will return the body
|
||||
as text (unicode object in Python 2 and str in Python 3)
|
||||
as str
|
||||
"""
|
||||
raise AttributeError("Response content isn't text")
|
||||
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class HtmlResponse(TextResponse):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
|
|||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import parsel
|
||||
from w3lib.encoding import html_to_unicode, resolve_encoding, \
|
||||
|
|
@ -16,7 +15,7 @@ from w3lib.html import strip_html5_whitespace
|
|||
from scrapy.http.request import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_native_str
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
||||
|
||||
class TextResponse(Response):
|
||||
|
|
@ -31,17 +30,14 @@ class TextResponse(Response):
|
|||
super(TextResponse, self).__init__(*args, **kwargs)
|
||||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, six.text_type):
|
||||
if six.PY2 and self.encoding is None:
|
||||
raise TypeError("Cannot convert unicode url - %s "
|
||||
"has no encoding" % type(self).__name__)
|
||||
self._url = to_native_str(url, self.encoding)
|
||||
if isinstance(url, str):
|
||||
self._url = to_unicode(url, self.encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_url(url)
|
||||
|
||||
def _set_body(self, body):
|
||||
self._body = b'' # used by encoding detection
|
||||
if isinstance(body, six.text_type):
|
||||
if isinstance(body, str):
|
||||
if self._encoding is None:
|
||||
raise TypeError('Cannot convert unicode body - %s has no encoding' %
|
||||
type(self).__name__)
|
||||
|
|
@ -84,11 +80,11 @@ class TextResponse(Response):
|
|||
@memoizemethod_noargs
|
||||
def _headers_encoding(self):
|
||||
content_type = self.headers.get(b'Content-Type', b'')
|
||||
return http_content_type_encoding(to_native_str(content_type))
|
||||
return http_content_type_encoding(to_unicode(content_type))
|
||||
|
||||
def _body_inferred_encoding(self):
|
||||
if self._cached_benc is None:
|
||||
content_type = to_native_str(self.headers.get(b'Content-Type', b''))
|
||||
content_type = to_unicode(self.headers.get(b'Content-Type', b''))
|
||||
benc, ubody = html_to_unicode(content_type, self.body,
|
||||
auto_detect_fun=self._auto_detect_fun,
|
||||
default_encoding=self._DEFAULT_ENCODING)
|
||||
|
|
@ -161,7 +157,7 @@ class TextResponse(Response):
|
|||
|
||||
def _url_from_selector(sel):
|
||||
# type: (parsel.Selector) -> str
|
||||
if isinstance(sel.root, six.string_types):
|
||||
if isinstance(sel.root, str):
|
||||
# e.g. ::attr(href) result
|
||||
return strip_html5_whitespace(sel.root)
|
||||
if not hasattr(sel.root, 'tag'):
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class XmlResponse(TextResponse):
|
||||
pass
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue