mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into response_follow_all
This commit is contained in:
Eugenio Lacuesta
commit
48223c107a
|
|
@ -0,0 +1,16 @@
|
|||
skips:
|
||||
- B101
|
||||
- B105
|
||||
- B303
|
||||
- B306
|
||||
- B307
|
||||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B402
|
||||
- B404
|
||||
- B406
|
||||
- B410
|
||||
- B503
|
||||
- B603
|
||||
- B605
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 1.7.0
|
||||
current_version = 1.8.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
27
.travis.yml
27
.travis.yml
|
|
@ -7,14 +7,10 @@ branches:
|
|||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=py27
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-pinned
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-extra-deps
|
||||
python: 2.7
|
||||
- env: TOXENV=pypy
|
||||
python: 2.7
|
||||
- env: TOXENV=security
|
||||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pypy3
|
||||
python: 3.5
|
||||
- env: TOXENV=py35
|
||||
|
|
@ -25,19 +21,14 @@ matrix:
|
|||
python: 3.6
|
||||
- env: TOXENV=py37
|
||||
python: 3.7
|
||||
- env: TOXENV=py37-extra-deps
|
||||
python: 3.7
|
||||
- env: TOXENV=py38
|
||||
python: 3.8
|
||||
- env: TOXENV=py38-extra-deps
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.6
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy" ]; then
|
||||
export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
|
|
@ -68,4 +59,4 @@ deploy:
|
|||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ members of the project's leadership.
|
|||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at [http://contributor-covenant.org/version/1/4][version]
|
||||
available at [http://contributor-covenant.org/version/1/4][version].
|
||||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
|
|
|
|||
20
README.rst
20
README.rst
|
|
@ -34,13 +34,13 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
|
|||
crawl websites and extract structured data from their pages. It can be used for
|
||||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
For more information including a list of features check the Scrapy homepage at:
|
||||
https://scrapy.org
|
||||
Check the Scrapy homepage at https://scrapy.org for more information,
|
||||
including a list of features.
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* Python 2.7 or Python 3.5+
|
||||
* Python 3.5+
|
||||
* Works on Linux, Windows, Mac OSX, BSD
|
||||
|
||||
Install
|
||||
|
|
@ -50,8 +50,8 @@ The quick way::
|
|||
|
||||
pip install scrapy
|
||||
|
||||
For more details see the install section in the documentation:
|
||||
https://docs.scrapy.org/en/latest/intro/install.html
|
||||
See the install section in the documentation at
|
||||
https://docs.scrapy.org/en/latest/intro/install.html for more details.
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
|
@ -62,17 +62,17 @@ directory.
|
|||
Releases
|
||||
========
|
||||
|
||||
You can find release notes at https://docs.scrapy.org/en/latest/news.html
|
||||
You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
|
||||
|
||||
Community (blog, twitter, mail list, IRC)
|
||||
=========================================
|
||||
|
||||
See https://scrapy.org/community/
|
||||
See https://scrapy.org/community/ for details.
|
||||
|
||||
Contributing
|
||||
============
|
||||
|
||||
See https://docs.scrapy.org/en/master/contributing.html
|
||||
See https://docs.scrapy.org/en/master/contributing.html for details.
|
||||
|
||||
Code of Conduct
|
||||
---------------
|
||||
|
|
@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
|
|||
Companies using Scrapy
|
||||
======================
|
||||
|
||||
See https://scrapy.org/companies/
|
||||
See https://scrapy.org/companies/ for a list.
|
||||
|
||||
Commercial Support
|
||||
==================
|
||||
|
||||
See https://scrapy.org/support/
|
||||
See https://scrapy.org/support/ for details.
|
||||
|
|
|
|||
10
conftest.py
10
conftest.py
|
|
@ -19,3 +19,13 @@ if six.PY3:
|
|||
def chdir(tmpdir):
|
||||
"""Change to pytest-provided temporary directory"""
|
||||
tmpdir.chdir()
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(session, config, items):
|
||||
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
|
||||
try:
|
||||
from pytest_flake8 import Flake8Item
|
||||
if config.getoption('--flake8'):
|
||||
items[:] = [item for item in items if isinstance(item, Flake8Item)]
|
||||
except ImportError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1,281 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Quotes to Scrape</title>
|
||||
<link rel="stylesheet" href="/static/bootstrap.min.css">
|
||||
<link rel="stylesheet" href="/static/main.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="row header-box">
|
||||
<div class="col-md-8">
|
||||
<h1>
|
||||
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
|
||||
</h1>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<p>
|
||||
|
||||
<a href="/login">Login</a>
|
||||
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-8">
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
|
||||
|
||||
<a class="tag" href="/tag/change/page/1/">change</a>
|
||||
|
||||
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
|
||||
|
||||
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
|
||||
|
||||
<a class="tag" href="/tag/world/page/1/">world</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
|
||||
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
|
||||
<a href="/author/J-K-Rowling">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
|
||||
|
||||
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
|
||||
|
||||
<a class="tag" href="/tag/choices/page/1/">choices</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||
|
||||
<a class="tag" href="/tag/live/page/1/">live</a>
|
||||
|
||||
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
|
||||
|
||||
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
|
||||
<span>by <small class="author" itemprop="author">Jane Austen</small>
|
||||
<a href="/author/Jane-Austen">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
|
||||
|
||||
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
|
||||
|
||||
<a class="tag" href="/tag/books/page/1/">books</a>
|
||||
|
||||
<a class="tag" href="/tag/classic/page/1/">classic</a>
|
||||
|
||||
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</span>
|
||||
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
|
||||
<a href="/author/Marilyn-Monroe">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
|
||||
|
||||
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
|
||||
|
||||
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
|
||||
|
||||
<a class="tag" href="/tag/success/page/1/">success</a>
|
||||
|
||||
<a class="tag" href="/tag/value/page/1/">value</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
|
||||
<span>by <small class="author" itemprop="author">André Gide</small>
|
||||
<a href="/author/Andre-Gide">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="life,love" / >
|
||||
|
||||
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||
|
||||
<a class="tag" href="/tag/love/page/1/">love</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“I have not failed. I've just found 10,000 ways that won't work.”</span>
|
||||
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
|
||||
<a href="/author/Thomas-A-Edison">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
|
||||
|
||||
<a class="tag" href="/tag/edison/page/1/">edison</a>
|
||||
|
||||
<a class="tag" href="/tag/failure/page/1/">failure</a>
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</span>
|
||||
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
|
||||
<a href="/author/Eleanor-Roosevelt">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
|
||||
|
||||
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
|
||||
<span>by <small class="author" itemprop="author">Steve Martin</small>
|
||||
<a href="/author/Steve-Martin">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
|
||||
|
||||
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||
|
||||
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
|
||||
|
||||
<a class="tag" href="/tag/simile/page/1/">simile</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav>
|
||||
<ul class="pager">
|
||||
|
||||
|
||||
<li class="next">
|
||||
<a href="/page/2/">Next <span aria-hidden="true">→</span></a>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="col-md-4 tags-box">
|
||||
|
||||
<h2>Top Ten tags</h2>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
|
||||
</span>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<footer class="footer">
|
||||
<div class="container">
|
||||
<p class="text-muted">
|
||||
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
|
||||
</p>
|
||||
<p class="copyright">
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
|
||||
</p>
|
||||
</div>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -27,10 +27,12 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
|
|||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = [
|
||||
'notfound.extension',
|
||||
'scrapydocs',
|
||||
'sphinx.ext.autodoc',
|
||||
'sphinx.ext.coverage',
|
||||
'sphinx.ext.intersphinx',
|
||||
'sphinx.ext.viewcode',
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
|
|
@ -237,7 +239,7 @@ coverage_ignore_pyobjects = [
|
|||
r'\bContractsManager\b$',
|
||||
|
||||
# For default contracts we only want to document their general purpose in
|
||||
# their constructor, the methods they reimplement to achieve that purpose
|
||||
# their __init__ method, the methods they reimplement to achieve that purpose
|
||||
# should be irrelevant to developers using those contracts.
|
||||
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
|
||||
|
||||
|
|
@ -273,4 +275,5 @@ coverage_ignore_pyobjects = [
|
|||
|
||||
intersphinx_mapping = {
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/stable', None),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
import os
|
||||
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
||||
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from sybil import Sybil
|
||||
from sybil.parsers.codeblock import CodeBlockParser
|
||||
from sybil.parsers.doctest import DocTestParser
|
||||
from sybil.parsers.skip import skip
|
||||
|
||||
|
||||
def load_response(url, filename):
|
||||
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
|
||||
with open(input_path, 'rb') as input_file:
|
||||
return HtmlResponse(url, body=input_file.read())
|
||||
|
||||
|
||||
def setup(namespace):
|
||||
namespace['load_response'] = load_response
|
||||
|
||||
|
||||
pytest_collect_file = Sybil(
|
||||
parsers=[
|
||||
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||
CodeBlockParser(future_imports=['print_function']),
|
||||
skip,
|
||||
],
|
||||
pattern='*.rst',
|
||||
setup=setup,
|
||||
).pytest()
|
||||
|
|
@ -177,20 +177,19 @@ Documentation policies
|
|||
======================
|
||||
|
||||
For reference documentation of API members (classes, methods, etc.) use
|
||||
docstrings and make sure that the Sphinx documentation uses the autodoc_
|
||||
extension to pull the docstrings. API reference documentation should follow
|
||||
docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
|
||||
and it may provide short examples.
|
||||
docstrings and make sure that the Sphinx documentation uses the
|
||||
:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
|
||||
documentation should follow docstring conventions (`PEP 257`_) and be
|
||||
IDE-friendly: short, to the point, and it may provide short examples.
|
||||
|
||||
Other types of documentation, such as tutorials or topics, should be covered in
|
||||
files within the ``docs/`` directory. This includes documentation that is
|
||||
specific to an API member, but goes beyond API reference documentation.
|
||||
|
||||
In any case, if something is covered in a docstring, use the autodoc_
|
||||
extension to pull the docstring into the documentation instead of duplicating
|
||||
the docstring in files within the ``docs/`` directory.
|
||||
|
||||
.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
|
||||
In any case, if something is covered in a docstring, use the
|
||||
:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
|
||||
documentation instead of duplicating the docstring in files within the
|
||||
``docs/`` directory.
|
||||
|
||||
Tests
|
||||
=====
|
||||
|
|
|
|||
|
|
@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 2.7 and Python 3.5+
|
||||
Scrapy is supported under Python 3.5+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 2.6 support was dropped starting at Scrapy 0.20.
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
Python 2 support was dropped in Scrapy 2.0.
|
||||
|
||||
.. note::
|
||||
For Python 3 support on Windows, it is recommended to use
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Installation guide
|
|||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 2.7 and Python 3.5 or above
|
||||
Scrapy runs on Python 3.5 or above
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
|
|
@ -102,10 +102,8 @@ just like any other Python package.
|
|||
(See :ref:`platform-specific guides <intro-install-platform-notes>`
|
||||
below for non-Python dependencies that you may need to install beforehand).
|
||||
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
|
||||
|
||||
* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
|
||||
* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
|
||||
only supports Python 3, make sure you created a Python 3 virtualenv.
|
||||
|
||||
.. _virtualenv: https://virtualenv.pypa.io
|
||||
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
|
||||
|
|
@ -149,16 +147,12 @@ typically too old and slow to catch up with latest Scrapy.
|
|||
To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
|
||||
these dependencies::
|
||||
|
||||
sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
|
||||
- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
|
||||
- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
|
||||
are required for ``lxml``
|
||||
- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
|
||||
|
||||
If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
|
||||
|
||||
sudo apt-get install python3 python3-dev
|
||||
|
||||
Inside a :ref:`virtualenv <intro-using-virtualenv>`,
|
||||
you can install Scrapy with ``pip`` after that::
|
||||
|
||||
|
|
@ -290,5 +284,5 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
|
|||
.. _zsh: https://www.zsh.org/
|
||||
.. _Scrapinghub: https://scrapinghub.com
|
||||
.. _Anaconda: https://docs.anaconda.com/anaconda/
|
||||
.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
|
||||
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
|
|
|
|||
|
|
@ -78,9 +78,9 @@ Our first Spider
|
|||
|
||||
Spiders are classes that you define and that Scrapy uses to scrape information
|
||||
from a website (or a group of websites). They must subclass
|
||||
:class:`scrapy.Spider` and define the initial requests to make, optionally how
|
||||
to follow links in the pages, and how to parse the downloaded page content to
|
||||
extract data.
|
||||
:class:`~scrapy.spiders.Spider` and define the initial requests to make,
|
||||
optionally how to follow links in the pages, and how to parse the downloaded
|
||||
page content to extract data.
|
||||
|
||||
This is the code for our first Spider. Save it in a file named
|
||||
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
|
||||
|
|
@ -235,13 +235,16 @@ You will see something like::
|
|||
[s] shelp() Shell help (print this help)
|
||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||
[s] view(response) View response in a browser
|
||||
>>>
|
||||
|
||||
Using the shell, you can try selecting elements using `CSS`_ with the response
|
||||
object::
|
||||
object:
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
||||
The result of running ``response.css('title')`` is a list-like object called
|
||||
:class:`~scrapy.selector.SelectorList`, which represents a list of
|
||||
|
|
@ -372,6 +375,9 @@ we want::
|
|||
We get a list of selectors for the quote HTML elements with::
|
||||
|
||||
>>> response.css("div.quote")
|
||||
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||
...]
|
||||
|
||||
Each of the selectors returned by the query above allows us to run further
|
||||
queries over their sub-elements. Let's assign the first selector to a
|
||||
|
|
@ -396,6 +402,12 @@ to get all of them::
|
|||
>>> tags
|
||||
['change', 'deep-thoughts', 'thinking', 'world']
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from sys import version_info
|
||||
|
||||
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
|
||||
|
||||
Having figured out how to extract each bit, we can now iterate over all the
|
||||
quotes elements and put them together into a Python dictionary::
|
||||
|
||||
|
|
@ -404,10 +416,9 @@ quotes elements and put them together into a Python dictionary::
|
|||
... author = quote.css("small.author::text").get()
|
||||
... tags = quote.css("div.tags a.tag::text").getall()
|
||||
... print(dict(text=text, author=author, tags=tags))
|
||||
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
|
||||
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
|
||||
... a few more of these, omitted for brevity
|
||||
>>>
|
||||
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
|
||||
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
|
||||
...
|
||||
|
||||
Extracting data in our spider
|
||||
-----------------------------
|
||||
|
|
@ -521,7 +532,7 @@ There is also an ``attrib`` property available
|
|||
(see :ref:`selecting-attributes` for more)::
|
||||
|
||||
>>> response.css('li.next a').attrib['href']
|
||||
'/page/2'
|
||||
'/page/2/'
|
||||
|
||||
Let's see now our spider modified to recursively follow the link to the next
|
||||
page, extracting data from it::
|
||||
|
|
|
|||
292
docs/news.rst
292
docs/news.rst
|
|
@ -6,22 +6,246 @@ Release notes
|
|||
.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
|
||||
planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
|
||||
|
||||
.. _release-1.8.0:
|
||||
|
||||
Scrapy 1.8.0 (2019-10-28)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* Dropped Python 3.4 support and updated minimum requirements; made Python 3.8
|
||||
support official
|
||||
* New :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class method
|
||||
* New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings
|
||||
* New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
|
||||
:setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Python 3.4 is no longer supported, and some of the minimum requirements of
|
||||
Scrapy have also changed:
|
||||
|
||||
* cssselect_ 0.9.1
|
||||
* cryptography_ 2.0
|
||||
* lxml_ 3.5.0
|
||||
* pyOpenSSL_ 16.2.0
|
||||
* queuelib_ 1.4.2
|
||||
* service_identity_ 16.0.0
|
||||
* six_ 1.10.0
|
||||
* Twisted_ 17.9.0 (16.0.0 with Python 2)
|
||||
* zope.interface_ 4.1.3
|
||||
|
||||
(:issue:`3892`)
|
||||
|
||||
* ``JSONRequest`` is now called :class:`~scrapy.http.JsonRequest` for
|
||||
consistency with similar classes (:issue:`3929`, :issue:`3982`)
|
||||
|
||||
* If you are using a custom context factory
|
||||
(:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
|
||||
accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
|
||||
(:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
|
||||
|
||||
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
|
||||
into lists::
|
||||
|
||||
>>> item = MyItem()
|
||||
>>> item['field'] = 'value1'
|
||||
>>> loader = ItemLoader(item=item)
|
||||
>>> item['field']
|
||||
['value1']
|
||||
|
||||
This is needed to allow adding values to existing fields
|
||||
(``loader.add_value('field', 'value2')``).
|
||||
|
||||
(:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
|
||||
:issue:`4036`)
|
||||
|
||||
See also :ref:`1.8-deprecation-removals` below.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* A new :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class
|
||||
method allows :ref:`creating a request from a cURL command
|
||||
<requests-from-curl>` (:issue:`2985`, :issue:`3862`)
|
||||
|
||||
* A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
|
||||
parser to use. It includes built-in support for
|
||||
:ref:`RobotFileParser <python-robotfileparser>`,
|
||||
:ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and
|
||||
:ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to
|
||||
:ref:`implement support for additional parsers
|
||||
<support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`,
|
||||
:issue:`3796`, :issue:`3935`, :issue:`3969`, :issue:`4006`)
|
||||
|
||||
* A new :setting:`ROBOTSTXT_USER_AGENT` setting allows defining a separate
|
||||
user agent string to use for robots.txt_ parsing (:issue:`3931`,
|
||||
:issue:`3966`)
|
||||
|
||||
* :class:`~scrapy.spiders.Rule` no longer requires a :class:`LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` parameter
|
||||
(:issue:`781`, :issue:`4016`)
|
||||
|
||||
* Use the new :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting to customize
|
||||
the TLS/SSL ciphers used by the default HTTP/1.1 downloader (:issue:`3392`,
|
||||
:issue:`3442`)
|
||||
|
||||
* Set the new :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` setting to
|
||||
``True`` to enable debug-level messages about TLS connection parameters
|
||||
after establishing HTTPS connections (:issue:`2111`, :issue:`3450`)
|
||||
|
||||
* Callbacks that receive keyword arguments
|
||||
(see :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>`) can now be
|
||||
tested using the new :class:`@cb_kwargs
|
||||
<scrapy.contracts.default.CallbackKeywordArgumentsContract>`
|
||||
:ref:`spider contract <topics-contracts>` (:issue:`3985`, :issue:`3988`)
|
||||
|
||||
* When a :class:`@scrapes <scrapy.contracts.default.ScrapesContract>` spider
|
||||
contract fails, all missing fields are now reported (:issue:`766`,
|
||||
:issue:`3939`)
|
||||
|
||||
* :ref:`Custom log formats <custom-log-formats>` can now drop messages by
|
||||
having the corresponding methods of the configured :setting:`LOG_FORMATTER`
|
||||
return ``None`` (:issue:`3984`, :issue:`3987`)
|
||||
|
||||
* A much improved completion definition is now available for Zsh_
|
||||
(:issue:`4069`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` no
|
||||
longer makes later calls to :meth:`ItemLoader.get_output_value()
|
||||
<scrapy.loader.ItemLoader.get_output_value>` or
|
||||
:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` return
|
||||
empty data (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`,
|
||||
:issue:`3998`, :issue:`4036`)
|
||||
|
||||
* Fixed :class:`~scrapy.statscollectors.DummyStatsCollector` raising a
|
||||
:exc:`TypeError` exception (:issue:`4007`, :issue:`4052`)
|
||||
|
||||
* :meth:`FilesPipeline.file_path
|
||||
<scrapy.pipelines.files.FilesPipeline.file_path>` and
|
||||
:meth:`ImagesPipeline.file_path
|
||||
<scrapy.pipelines.images.ImagesPipeline.file_path>` no longer choose
|
||||
file extensions that are not `registered with IANA`_ (:issue:`1287`,
|
||||
:issue:`3953`, :issue:`3954`)
|
||||
|
||||
* When using botocore_ to persist files in S3, all botocore-supported headers
|
||||
are properly mapped now (:issue:`3904`, :issue:`3905`)
|
||||
|
||||
* FTP passwords in :setting:`FEED_URI` containing percent-escaped characters
|
||||
are now properly decoded (:issue:`3941`)
|
||||
|
||||
* A memory-handling and error-handling issue in
|
||||
:func:`scrapy.utils.ssl.get_temp_key_info` has been fixed (:issue:`3920`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* The documentation now covers how to define and configure a :ref:`custom log
|
||||
format <custom-log-formats>` (:issue:`3616`, :issue:`3660`)
|
||||
|
||||
* API documentation added for :class:`~scrapy.exporters.MarshalItemExporter`
|
||||
and :class:`~scrapy.exporters.PythonItemExporter` (:issue:`3973`)
|
||||
|
||||
* API documentation added for :class:`~scrapy.item.BaseItem` and
|
||||
:class:`~scrapy.item.ItemMeta` (:issue:`3999`)
|
||||
|
||||
* Minor documentation fixes (:issue:`2998`, :issue:`3398`, :issue:`3597`,
|
||||
:issue:`3894`, :issue:`3934`, :issue:`3978`, :issue:`3993`, :issue:`4022`,
|
||||
:issue:`4028`, :issue:`4033`, :issue:`4046`, :issue:`4050`, :issue:`4055`,
|
||||
:issue:`4056`, :issue:`4061`, :issue:`4072`, :issue:`4071`, :issue:`4079`,
|
||||
:issue:`4081`, :issue:`4089`, :issue:`4093`)
|
||||
|
||||
|
||||
.. _1.8-deprecation-removals:
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* ``scrapy.xlib`` has been removed (:issue:`4015`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* The LevelDB_ storage backend
|
||||
(``scrapy.extensions.httpcache.LeveldbCacheStorage``) of
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is
|
||||
deprecated (:issue:`4085`, :issue:`4092`)
|
||||
|
||||
* Use of the undocumented ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
|
||||
variable is deprecated (:issue:`3910`)
|
||||
|
||||
* ``scrapy.item.DictItem`` is deprecated, use :class:`~scrapy.item.Item`
|
||||
instead (:issue:`3999`)
|
||||
|
||||
|
||||
Other changes
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Minimum versions of optional Scrapy requirements that are covered by
|
||||
continuous integration tests have been updated:
|
||||
|
||||
* botocore_ 1.3.23
|
||||
* Pillow_ 3.4.2
|
||||
|
||||
Lower versions of these optional requirements may work, but it is not
|
||||
guaranteed (:issue:`3892`)
|
||||
|
||||
* GitHub templates for bug reports and feature requests (:issue:`3126`,
|
||||
:issue:`3471`, :issue:`3749`, :issue:`3754`)
|
||||
|
||||
* Continuous integration fixes (:issue:`3923`)
|
||||
|
||||
* Code cleanup (:issue:`3391`, :issue:`3907`, :issue:`3946`, :issue:`3950`,
|
||||
:issue:`4023`, :issue:`4031`)
|
||||
|
||||
|
||||
.. _release-1.7.4:
|
||||
|
||||
Scrapy 1.7.4 (2019-10-21)
|
||||
-------------------------
|
||||
|
||||
Revert the fix for :issue:`3804` (:issue:`3819`), which has a few undesired
|
||||
side effects (:issue:`3897`, :issue:`3976`).
|
||||
|
||||
As a result, when an item loader is initialized with an item,
|
||||
:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` once again
|
||||
makes later calls to :meth:`ItemLoader.get_output_value()
|
||||
<scrapy.loader.ItemLoader.get_output_value>` or :meth:`ItemLoader.load_item()
|
||||
<scrapy.loader.ItemLoader.load_item>` return empty data.
|
||||
|
||||
|
||||
.. _release-1.7.3:
|
||||
|
||||
Scrapy 1.7.3 (2019-08-01)
|
||||
-------------------------
|
||||
|
||||
Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
|
||||
|
||||
|
||||
.. _release-1.7.2:
|
||||
|
||||
Scrapy 1.7.2 (2019-07-23)
|
||||
-------------------------
|
||||
|
||||
Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
|
||||
|
||||
|
||||
.. _release-1.7.1:
|
||||
|
||||
Scrapy 1.7.1 (2019-07-18)
|
||||
-------------------------
|
||||
|
||||
Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
|
||||
|
||||
|
||||
.. _release-1.7.0:
|
||||
|
||||
Scrapy 1.7.0 (2019-07-18)
|
||||
|
|
@ -71,7 +295,7 @@ New features
|
|||
~~~~~~~~~~~~
|
||||
|
||||
* A new scheduler priority queue,
|
||||
:class:`scrapy.pqueues.DownloaderAwarePriorityQueue`, may be
|
||||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targetting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
|
@ -84,12 +308,12 @@ New features
|
|||
convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
|
||||
|
||||
* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
|
||||
constructor now receives the :class:`~scrapy.http.Response` object that
|
||||
``__init__`` method now receives the :class:`~scrapy.http.Response` object that
|
||||
originated the request as its second argument (:issue:`3682`)
|
||||
|
||||
* A new ``restrict_text`` parameter for the
|
||||
:attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
constructor allows filtering links by linking text (:issue:`3622`,
|
||||
``__init__`` method allows filtering links by linking text (:issue:`3622`,
|
||||
:issue:`3635`)
|
||||
|
||||
* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
|
||||
|
|
@ -150,9 +374,9 @@ Bug fixes
|
|||
:setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
|
||||
(:issue:`3625`)
|
||||
|
||||
* Fixed a memory leak in :class:`~scrapy.pipelines.media.MediaPipeline`
|
||||
affecting, for example, non-200 responses and exceptions from custom
|
||||
middlewares (:issue:`3813`)
|
||||
* Fixed a memory leak in ``scrapy.pipelines.media.MediaPipeline`` affecting,
|
||||
for example, non-200 responses and exceptions from custom middlewares
|
||||
(:issue:`3813`)
|
||||
|
||||
* Requests with private callbacks are now correctly unserialized from disk
|
||||
(:issue:`3790`)
|
||||
|
|
@ -255,7 +479,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
|
|||
|
||||
* From :class:`~scrapy.selector.Selector`:
|
||||
|
||||
* ``_root`` (both the constructor argument and the object property, use
|
||||
* ``_root`` (both the ``__init__`` method argument and the object property, use
|
||||
``root``)
|
||||
|
||||
* ``extract_unquoted`` (use ``getall``)
|
||||
|
|
@ -301,7 +525,7 @@ Deprecations
|
|||
|
||||
* The ``queuelib.PriorityQueue`` value for the
|
||||
:setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
|
||||
:class:`scrapy.pqueues.ScrapyPriorityQueue` instead.
|
||||
``scrapy.pqueues.ScrapyPriorityQueue`` instead.
|
||||
|
||||
* ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
|
||||
do not accept two arguments are deprecated.
|
||||
|
|
@ -551,12 +775,12 @@ Scrapy 1.5.2 (2019-01-22)
|
|||
|
||||
*The fix is backward incompatible*, it enables telnet user-password
|
||||
authentication by default with a random generated password. If you can't
|
||||
upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT`
|
||||
upgrade right away, please consider setting :setting:`TELNETCONSOLE_PORT`
|
||||
out of its default value.
|
||||
|
||||
See :ref:`telnet console <topics-telnetconsole>` documentation for more info
|
||||
|
||||
* Backport CI build failure under GCE environemnt due to boto import error.
|
||||
* Backport CI build failure under GCE environment due to boto import error.
|
||||
|
||||
.. _release-1.5.1:
|
||||
|
||||
|
|
@ -758,7 +982,9 @@ Enjoy! (Or read on for the rest of changes in this release.)
|
|||
Deprecations and Backward Incompatible Changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
|
||||
- Default to ``canonicalize=False`` in
|
||||
:class:`scrapy.linkextractors.LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
(:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
|
||||
**warning, this is technically backward-incompatible**
|
||||
- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
|
||||
|
|
@ -794,10 +1020,13 @@ New Features
|
|||
- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
|
||||
- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
|
||||
- Warn users when project contains duplicate spider names (fixes :issue:`2181`)
|
||||
- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
|
||||
- :ref:`Media downloads <topics-media-pipeline>`, with :class:`FilesPipelines`
|
||||
or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
|
||||
using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
|
||||
- ``scrapy.utils.datatypes.CaselessDict`` now accepts ``Mapping`` instances and
|
||||
not only dicts (:issue:`2646`)
|
||||
- :ref:`Media downloads <topics-media-pipeline>`, with
|
||||
:class:`~scrapy.pipelines.files.FilesPipeline` or
|
||||
:class:`~scrapy.pipelines.images.ImagesPipeline`, can now optionally handle
|
||||
HTTP redirects using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting
|
||||
(:issue:`2616`, fixes :issue:`2004`)
|
||||
- Accept non-complete responses from websites using a new
|
||||
:setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
|
||||
- Optional pretty-printing of JSON and XML items via
|
||||
|
|
@ -817,8 +1046,8 @@ Bug fixes
|
|||
|
||||
- LinkExtractor now strips leading and trailing whitespaces from attributes
|
||||
(:issue:`2547`, fixes :issue:`1614`)
|
||||
- Properly handle whitespaces in action attribute in :class:`FormRequest`
|
||||
(:issue:`2548`)
|
||||
- Properly handle whitespaces in action attribute in
|
||||
:class:`~scrapy.http.FormRequest` (:issue:`2548`)
|
||||
- Buffer CONNECT response bytes from proxy until all HTTP headers are received
|
||||
(:issue:`2495`, fixes :issue:`2491`)
|
||||
- FTP downloader now works on Python 3, provided you use Twisted>=17.1
|
||||
|
|
@ -851,7 +1080,8 @@ Cleanups & Refactoring
|
|||
fixes :issue:`2560`)
|
||||
- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
|
||||
- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
|
||||
- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
|
||||
- Catch more specific ``os.error`` exception in
|
||||
``scrapy.pipelines.files.FSFilesStore`` (:issue:`2644`)
|
||||
- Change "localhost" test server certificate (:issue:`2720`)
|
||||
- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
|
||||
|
||||
|
|
@ -868,7 +1098,8 @@ Documentation
|
|||
(:issue:`2477`, fixes :issue:`2475`)
|
||||
- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
|
||||
- Rearrange selector sections (:issue:`2705`)
|
||||
- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
|
||||
- Remove ``__nonzero__`` from :class:`~scrapy.selector.SelectorList`
|
||||
docs (:issue:`2683`)
|
||||
- Mention how to disable request filtering in documentation of
|
||||
:setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
|
||||
- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
|
||||
|
|
@ -2327,7 +2558,7 @@ Scrapy 0.18.0 (released 2013-08-09)
|
|||
- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
|
||||
- Add scrapy commands using external libraries (:issue:`260`)
|
||||
- Added ``--pdb`` option to ``scrapy`` command line tool
|
||||
- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
|
||||
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
|
||||
- Several improvements to spider contracts
|
||||
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
|
||||
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
|
||||
|
|
@ -2448,7 +2679,7 @@ Scrapy changes:
|
|||
- added options ``-o`` and ``-t`` to the :command:`runspider` command
|
||||
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
|
||||
- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
|
||||
- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
|
||||
- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
|
||||
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
- dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
|
||||
|
|
@ -2472,7 +2703,7 @@ Scrapy changes:
|
|||
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
|
||||
- promoted :ref:`topics-djangoitem` to main contrib
|
||||
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
|
||||
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
|
||||
- removed signal: ``scrapy.mail.mail_sent``
|
||||
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
|
||||
|
|
@ -2609,7 +2840,8 @@ The numbers like #NNN reference tickets in the old issue tracker (Trac) which is
|
|||
New features and improvements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273)
|
||||
- Passed item is now sent in the ``item`` argument of the :signal:`item_passed
|
||||
<item_scraped>` (#273)
|
||||
- Added verbose option to ``scrapy version`` command, useful for bug reports (#298)
|
||||
- HTTP cache now stored by default in the project data dir (#279)
|
||||
- Added project data storage directory (#276, #277)
|
||||
|
|
@ -2685,7 +2917,7 @@ API changes
|
|||
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
|
||||
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
|
||||
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
|
||||
- Changes to Scrapy Manager (now called "Crawler"):
|
||||
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
|
||||
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
|
||||
|
|
@ -2810,23 +3042,35 @@ First release of Scrapy.
|
|||
|
||||
|
||||
.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
|
||||
.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
|
||||
.. _cryptography: https://cryptography.io/en/latest/
|
||||
.. _cssselect: https://github.com/scrapy/cssselect/
|
||||
.. _docstrings: https://docs.python.org/glossary.html#term-docstring
|
||||
.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
|
||||
.. _LevelDB: https://github.com/google/leveldb
|
||||
.. _lxml: http://lxml.de/
|
||||
.. _marshal: https://docs.python.org/2/library/marshal.html
|
||||
.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
|
||||
.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
|
||||
.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
|
||||
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
|
||||
.. _Pillow: https://python-pillow.org/
|
||||
.. _pyOpenSSL: https://www.pyopenssl.org/en/stable/
|
||||
.. _queuelib: https://github.com/scrapy/queuelib
|
||||
.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
|
||||
.. _resource: https://docs.python.org/2/library/resource.html
|
||||
.. _robots.txt: http://www.robotstxt.org/
|
||||
.. _scrapely: https://github.com/scrapy/scrapely
|
||||
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
|
||||
.. _six: https://six.readthedocs.io/
|
||||
.. _tox: https://pypi.python.org/pypi/tox
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _w3lib: https://github.com/scrapy/w3lib
|
||||
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
|
||||
.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
|
||||
.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
|
||||
.. _Zsh: https://www.zsh.org/
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
Sphinx>=2.1
|
||||
sphinx_rtd_theme
|
||||
sphinx-notfound-page
|
||||
sphinx_rtd_theme
|
||||
|
|
|
|||
|
|
@ -6,10 +6,6 @@ Spiders Contracts
|
|||
|
||||
.. versionadded:: 0.15
|
||||
|
||||
.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
|
||||
to minor functionality/API updates. Check the :ref:`release notes <news>` to
|
||||
be notified of updates.
|
||||
|
||||
Testing spiders can get particularly annoying and while nothing prevents you
|
||||
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
|
||||
integrated way of testing your spiders by the means of contracts.
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ where our quotes are coming from:
|
|||
First click on the request with the name ``scroll``. On the right
|
||||
you can now inspect the request. In ``Headers`` you'll find details
|
||||
about the request headers, such as the URL, the method, the IP-address,
|
||||
and so on. We'll ignore the other tabs and click directly on ``Reponse``.
|
||||
and so on. We'll ignore the other tabs and click directly on ``Response``.
|
||||
|
||||
What you should see in the ``Preview`` pane is the rendered HTML-code,
|
||||
that is exactly what we saw when we called ``view(response)`` in the
|
||||
|
|
|
|||
|
|
@ -348,7 +348,6 @@ HttpCacheMiddleware
|
|||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
* :ref:`httpcache-storage-leveldb`
|
||||
|
||||
You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
|
||||
setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
|
||||
|
|
@ -475,30 +474,9 @@ DBM storage backend
|
|||
|
||||
A DBM_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
By default, it uses the anydbm_ module, but you can change it with the
|
||||
By default, it uses the :mod:`dbm`, but you can change it with the
|
||||
:setting:`HTTPCACHE_DBM_MODULE` setting.
|
||||
|
||||
.. _httpcache-storage-leveldb:
|
||||
|
||||
LevelDB storage backend
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. class:: LeveldbCacheStorage
|
||||
|
||||
.. versionadded:: 0.23
|
||||
|
||||
A LevelDB_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
This backend is not recommended for development because only one process
|
||||
can access LevelDB databases at the same time, so you can't run a crawl and
|
||||
open the scrapy shell in parallel for the same spider.
|
||||
|
||||
In order to use this storage backend, install the `LevelDB python
|
||||
bindings`_ (e.g. ``pip install leveldb``).
|
||||
|
||||
.. _LevelDB: https://github.com/google/leveldb
|
||||
.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
|
||||
|
||||
.. _httpcache-storage-custom:
|
||||
|
||||
Writing your own storage backend
|
||||
|
|
@ -534,7 +512,7 @@ defines the methods described below.
|
|||
:param spider: the spider which generated the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
:param request: the request to find cached reponse for
|
||||
:param request: the request to find cached response for
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
||||
.. method:: store_response(spider, request, response)
|
||||
|
|
@ -648,7 +626,7 @@ HTTPCACHE_DBM_MODULE
|
|||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``'anydbm'``
|
||||
Default: ``'dbm'``
|
||||
|
||||
The database module to use in the :ref:`DBM storage backend
|
||||
<httpcache-storage-dbm>`. This setting is specific to the DBM backend.
|
||||
|
|
@ -1224,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
|
|||
|
||||
|
||||
.. _DBM: https://en.wikipedia.org/wiki/Dbm
|
||||
.. _anydbm: https://docs.python.org/2/library/anydbm.html
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ Quick example
|
|||
=============
|
||||
|
||||
There are two ways to instantiate the mail sender. You can instantiate it using
|
||||
the standard constructor::
|
||||
the standard ``__init__`` method::
|
||||
|
||||
from scrapy.mail import MailSender
|
||||
mailer = MailSender()
|
||||
|
|
@ -111,7 +111,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
|
|||
Mail settings
|
||||
=============
|
||||
|
||||
These settings define the default constructor values of the :class:`MailSender`
|
||||
These settings define the default ``__init__`` method values of the :class:`MailSender`
|
||||
class, and can be used to configure e-mail notifications in your project without
|
||||
writing any code (for those extensions and code that uses :class:`MailSender`).
|
||||
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ described next.
|
|||
1. Declaring a serializer in the field
|
||||
--------------------------------------
|
||||
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
a callable which receives a value and returns its serialized form.
|
||||
|
||||
Example::
|
||||
|
|
@ -144,7 +144,7 @@ BaseItemExporter
|
|||
defining what fields to export, whether to export empty fields, or which
|
||||
encoding to use.
|
||||
|
||||
These features can be configured through the constructor arguments which
|
||||
These features can be configured through the ``__init__`` method arguments which
|
||||
populate their respective instance attributes: :attr:`fields_to_export`,
|
||||
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
|
||||
|
||||
|
|
@ -246,8 +246,8 @@ XmlItemExporter
|
|||
:param item_element: The name of each item element in the exported XML.
|
||||
:type item_element: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -306,9 +306,9 @@ CsvItemExporter
|
|||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor, and the leftover arguments to the
|
||||
`csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
|
||||
argument to customize this exporter.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
|
@ -334,8 +334,8 @@ PickleItemExporter
|
|||
|
||||
For more information, refer to the `pickle module documentation`_.
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
Pickle isn't a human readable format, so no output examples are provided.
|
||||
|
||||
|
|
@ -351,8 +351,8 @@ PprintItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -367,10 +367,10 @@ JsonItemExporter
|
|||
.. class:: JsonItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional constructor arguments are
|
||||
passed to the :class:`BaseItemExporter` constructor, and the leftover
|
||||
arguments to the `JSONEncoder`_ constructor, so you can use any
|
||||
`JSONEncoder`_ constructor argument to customize this exporter.
|
||||
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
|
||||
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -398,10 +398,10 @@ JsonLinesItemExporter
|
|||
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional constructor arguments are passed
|
||||
to the :class:`BaseItemExporter` constructor, and the leftover arguments to
|
||||
the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
|
||||
constructor argument to customize this exporter.
|
||||
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
|
||||
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
|
||||
``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ Loading & activating extensions
|
|||
|
||||
Extensions are loaded and activated at startup by instantiating a single
|
||||
instance of the extension class. Therefore, all the extension initialization
|
||||
code must be performed in the class constructor (``__init__`` method).
|
||||
code must be performed in the class ``__init__`` method.
|
||||
|
||||
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
|
||||
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
|
||||
|
|
|
|||
|
|
@ -99,12 +99,12 @@ The storages backends supported out of the box are:
|
|||
|
||||
* :ref:`topics-feed-storage-fs`
|
||||
* :ref:`topics-feed-storage-ftp`
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
* :ref:`topics-feed-storage-stdout`
|
||||
|
||||
Some storage backends may be unavailable if the required external libraries are
|
||||
not available. For example, the S3 backend is only available if the botocore_
|
||||
or boto_ library is installed (Scrapy supports boto_ only on Python 2).
|
||||
library is installed.
|
||||
|
||||
|
||||
.. _topics-feed-uri-params:
|
||||
|
|
@ -182,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
|
|||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
|
||||
* Required external libraries: `botocore`_
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
|
@ -399,6 +399,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
|
||||
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _boto: https://github.com/boto/boto
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ especially in a larger project with many spiders.
|
|||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide a `dictionary-like`_ API with a convenient syntax for declaring
|
||||
their available fields.
|
||||
their available fields.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
serialization can be customized using Item fields metadata, :mod:`trackref`
|
||||
tracks Item instances to help find memory leaks
|
||||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
|
||||
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
|
@ -237,8 +237,12 @@ Item objects
|
|||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
|
||||
Items replicate the standard `dict API`_, including its constructor. The
|
||||
only additional attribute provided by Items is:
|
||||
Items replicate the standard `dict API`_, including its ``__init__`` method, and
|
||||
also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
||||
.. automethod:: deepcopy
|
||||
|
||||
.. attribute:: fields
|
||||
|
||||
|
|
|
|||
|
|
@ -71,34 +71,11 @@ on cookies.
|
|||
Request serialization
|
||||
---------------------
|
||||
|
||||
Requests must be serializable by the ``pickle`` module, in order for persistence
|
||||
to work, so you should make sure that your requests are serializable.
|
||||
|
||||
The most common issue here is to use ``lambda`` functions on request callbacks that
|
||||
can't be persisted.
|
||||
|
||||
So, for example, this won't work::
|
||||
|
||||
def some_callback(self, response):
|
||||
somearg = 'test'
|
||||
return scrapy.Request('http://www.example.com',
|
||||
callback=lambda r: self.other_callback(r, somearg))
|
||||
|
||||
def other_callback(self, response, somearg):
|
||||
print("the argument passed is: %s" % somearg)
|
||||
|
||||
But this will::
|
||||
|
||||
def some_callback(self, response):
|
||||
somearg = 'test'
|
||||
return scrapy.Request('http://www.example.com',
|
||||
callback=self.other_callback, cb_kwargs={'somearg': somearg})
|
||||
|
||||
def other_callback(self, response, somearg):
|
||||
print("the argument passed is: %s" % somearg)
|
||||
For persistence to work, :class:`~scrapy.http.Request` objects must be
|
||||
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
|
||||
values passed to their ``__init__`` method, which must be methods of the
|
||||
runnning :class:`~scrapy.spiders.Spider` class.
|
||||
|
||||
If you wish to log the requests that couldn't be serialized, you can set the
|
||||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||
It is ``False`` by default.
|
||||
|
||||
.. _pickle: https://docs.python.org/library/pickle.html
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ knowledge about Python internals. For more info about Guppy, refer to the
|
|||
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
If you're using Python 3, you can use muppy from `Pympler`_.
|
||||
You can use muppy from `Pympler`_.
|
||||
|
||||
.. _Pympler: https://pypi.org/project/Pympler/
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Using Item Loaders to populate items
|
|||
|
||||
To use an Item Loader, you must first instantiate it. You can either
|
||||
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
|
||||
which case an Item is automatically instantiated in the Item Loader constructor
|
||||
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
|
||||
using the Item class specified in the :attr:`ItemLoader.default_item_class`
|
||||
attribute.
|
||||
|
||||
|
|
@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
|
|||
the same item field; the Item Loader will know how to "join" those values later
|
||||
using a proper processing function.
|
||||
|
||||
.. note:: Collected data is internally stored as lists,
|
||||
allowing to add several values to the same field.
|
||||
If an ``item`` argument is passed when creating a loader,
|
||||
each of the item's values will be stored as-is if it's already
|
||||
an iterable, or wrapped with a list if it's a single value.
|
||||
|
||||
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
|
||||
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
|
||||
chapter <topics-items>`::
|
||||
|
|
@ -128,9 +134,9 @@ So what happens is:
|
|||
It's worth noticing that processors are just callable objects, which are called
|
||||
with the data to be parsed, and return a parsed value. So you can use any
|
||||
function as input or output processor. The only requirement is that they must
|
||||
accept one (and only one) positional argument, which will be an iterator.
|
||||
accept one (and only one) positional argument, which will be an iterable.
|
||||
|
||||
.. note:: Both input and output processors must receive an iterator as their
|
||||
.. note:: Both input and output processors must receive an iterable as their
|
||||
first argument. The output of those functions can be anything. The result of
|
||||
input processors will be appended to an internal list (in the Loader)
|
||||
containing the collected values (for that field). The result of the output
|
||||
|
|
@ -265,7 +271,7 @@ There are several ways to modify Item Loader context values:
|
|||
loader.context['unit'] = 'cm'
|
||||
|
||||
2. On Item Loader instantiation (the keyword arguments of Item Loader
|
||||
constructor are stored in the Item Loader context)::
|
||||
``__init__`` method are stored in the Item Loader context)::
|
||||
|
||||
loader = ItemLoader(product, unit='cm')
|
||||
|
||||
|
|
@ -494,7 +500,7 @@ ItemLoader objects
|
|||
.. attribute:: default_item_class
|
||||
|
||||
An Item class (or factory), used to instantiate items when not given in
|
||||
the constructor.
|
||||
the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
|
|
@ -509,15 +515,15 @@ ItemLoader objects
|
|||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the constructor.
|
||||
If a selector is given in the constructor this attribute is ignored.
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the constructor or one created from
|
||||
the response given in the constructor using the
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
|
||||
|
|
@ -642,7 +648,7 @@ Here is a list of all built-in processors:
|
|||
.. class:: Identity
|
||||
|
||||
The simplest processor, which doesn't do anything. It returns the original
|
||||
values unchanged. It doesn't receive any constructor arguments, nor does it
|
||||
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
|
||||
accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
|
@ -656,7 +662,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
Returns the first non-null/non-empty value from the values received,
|
||||
so it's typically used as an output processor to single-valued fields.
|
||||
It doesn't receive any constructor arguments, nor does it accept Loader contexts.
|
||||
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -667,7 +673,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
.. class:: Join(separator=u' ')
|
||||
|
||||
Returns the values joined with the separator given in the constructor, which
|
||||
Returns the values joined with the separator given in the ``__init__`` method, which
|
||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||
|
||||
When using the default separator, this processor is equivalent to the
|
||||
|
|
@ -705,7 +711,7 @@ Here is a list of all built-in processors:
|
|||
those which do, this processor will pass the currently active :ref:`Loader
|
||||
context <topics-loaders-context>` through that parameter.
|
||||
|
||||
The keyword arguments passed in the constructor are used as the default
|
||||
The keyword arguments passed in the ``__init__`` method are used as the default
|
||||
Loader context values passed to each function call. However, the final
|
||||
Loader context values passed to functions are overridden with the currently
|
||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
|
|
@ -749,12 +755,12 @@ Here is a list of all built-in processors:
|
|||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
constructor keyword arguments are used as default context values. See
|
||||
``__init__`` method keyword arguments are used as default context values. See
|
||||
:class:`Compose` processor for more info.
|
||||
|
||||
.. class:: SelectJmes(json_path)
|
||||
|
||||
Queries the value using the json path provided to the constructor and returns the output.
|
||||
Queries the value using the json path provided to the ``__init__`` method and returns the output.
|
||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||
This processor takes only one input at a time.
|
||||
|
||||
|
|
|
|||
|
|
@ -198,8 +198,9 @@ to override some of the Scrapy settings regarding logging.
|
|||
Custom Log Formats
|
||||
------------------
|
||||
|
||||
A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
|
||||
and making :setting:`LOG_FORMATTER` point to your new class.
|
||||
A custom log format can be set for different actions by extending
|
||||
:class:`~scrapy.logformatter.LogFormatter` class and making
|
||||
:setting:`LOG_FORMATTER` point to your new class.
|
||||
|
||||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
|
@ -254,18 +255,18 @@ scrapy.utils.log module
|
|||
when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
|
||||
In that case, its usage is not required but it's recommended.
|
||||
|
||||
If you plan on configuring the handlers yourself is still recommended you
|
||||
call this function, passing ``install_root_handler=False``. Bear in mind
|
||||
there won't be any log output set by default in that case.
|
||||
Another option when running custom scripts is to manually configure the logging.
|
||||
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
|
||||
|
||||
To get you started on manually configuring logging's output, you can use
|
||||
`logging.basicConfig()`_ to set a basic root handler. This is an example
|
||||
on how to redirect ``INFO`` or higher messages to a file::
|
||||
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
|
||||
so it is recommended to only use `logging.basicConfig()`_ together with
|
||||
:class:`~scrapy.crawler.CrawlerRunner`.
|
||||
|
||||
This is an example on how to redirect ``INFO`` or higher messages to a file::
|
||||
|
||||
import logging
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
configure_logging(install_root_handler=False)
|
||||
logging.basicConfig(
|
||||
filename='log.txt',
|
||||
format='%(levelname)s: %(message)s',
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ policy::
|
|||
|
||||
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
|
||||
|
||||
Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
|
||||
|
||||
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ Request objects
|
|||
|
||||
A string containing the URL of this request. Keep in mind that this
|
||||
attribute contains the escaped URL, so it can differ from the URL passed in
|
||||
the constructor.
|
||||
the ``__init__`` method.
|
||||
|
||||
This attribute is read-only. To change the URL of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
.. class:: FormRequest(url, [formdata, ...])
|
||||
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
|
|||
:type dont_click: boolean
|
||||
|
||||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` constructor.
|
||||
:class:`FormRequest` ``__init__`` method.
|
||||
|
||||
.. versionadded:: 0.10.3
|
||||
The ``formname`` parameter.
|
||||
|
|
@ -547,7 +547,7 @@ dealing with JSON requests.
|
|||
|
||||
.. class:: JsonRequest(url, [... data, dumps_kwargs])
|
||||
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -556,7 +556,7 @@ dealing with JSON requests.
|
|||
|
||||
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
|
||||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
|
|
@ -596,8 +596,8 @@ Response objects
|
|||
(for single valued headers) or lists (for multi-valued headers).
|
||||
:type headers: dict
|
||||
|
||||
:param body: the response body. To access the decoded text as str (unicode
|
||||
in Python 2) you can use ``response.text`` from an encoding-aware
|
||||
:param body: the response body. To access the decoded text as str you can use
|
||||
``response.text`` from an encoding-aware
|
||||
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
|
||||
such as :class:`TextResponse`.
|
||||
:type body: bytes
|
||||
|
|
@ -723,7 +723,7 @@ TextResponse objects
|
|||
:class:`Response` class, which is meant to be used only for binary data,
|
||||
such as images, sounds or any media file.
|
||||
|
||||
:class:`TextResponse` objects support a new constructor argument, in
|
||||
:class:`TextResponse` objects support a new ``__init__`` method argument, in
|
||||
addition to the base :class:`Response` objects. The remaining functionality
|
||||
is the same as for the :class:`Response` class and is not documented here.
|
||||
|
||||
|
|
@ -757,7 +757,7 @@ TextResponse objects
|
|||
A string with the encoding of this response. The encoding is resolved by
|
||||
trying the following mechanisms, in order:
|
||||
|
||||
1. the encoding passed in the constructor ``encoding`` argument
|
||||
1. the encoding passed in the ``__init__`` method ``encoding`` argument
|
||||
|
||||
2. the encoding declared in the Content-Type HTTP header. If this
|
||||
encoding is not valid (ie. unknown), it is ignored and the next
|
||||
|
|
|
|||
|
|
@ -188,7 +188,6 @@ AWS_ENDPOINT_URL
|
|||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
|
|
@ -199,7 +198,6 @@ Default: ``None``
|
|||
|
||||
Use this option if you want to disable SSL connection for communication with
|
||||
S3 or S3-like storage. By default SSL will be used.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: AWS_VERIFY
|
||||
|
||||
|
|
@ -209,7 +207,7 @@ AWS_VERIFY
|
|||
Default: ``None``
|
||||
|
||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur. Only supported with ``botocore`` library.
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
|
|
@ -219,7 +217,6 @@ AWS_REGION_NAME
|
|||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
|
|
@ -229,8 +226,7 @@ BOT_NAME
|
|||
Default: ``'scrapybot'``
|
||||
|
||||
The name of the bot implemented by this Scrapy project (also known as the
|
||||
project name). This will be used to construct the User-Agent by default, and
|
||||
also for logging.
|
||||
project name). This name will be used for the logging too.
|
||||
|
||||
It's automatically populated with your project name when you create your
|
||||
project with the :command:`startproject` command.
|
||||
|
|
@ -796,6 +792,7 @@ Default: ``True``
|
|||
|
||||
Whether or not to use passive mode when initiating FTP transfers.
|
||||
|
||||
.. reqmeta:: ftp_password
|
||||
.. setting:: FTP_PASSWORD
|
||||
|
||||
FTP_PASSWORD
|
||||
|
|
@ -814,6 +811,7 @@ in ``Request`` meta.
|
|||
|
||||
.. _RFC 1635: https://tools.ietf.org/html/rfc1635
|
||||
|
||||
.. reqmeta:: ftp_user
|
||||
.. setting:: FTP_USER
|
||||
|
||||
FTP_USER
|
||||
|
|
|
|||
|
|
@ -72,8 +72,6 @@ scrapy.Spider
|
|||
spider that crawls ``mywebsite.com`` would often be called
|
||||
``mywebsite``.
|
||||
|
||||
.. note:: In Python 2 this must be ASCII only.
|
||||
|
||||
.. attribute:: allowed_domains
|
||||
|
||||
An optional list of strings containing domains that this spider is
|
||||
|
|
|
|||
|
|
@ -1,25 +1,210 @@
|
|||
#compdef scrapy
|
||||
|
||||
# zsh completion for the Scrapy command-line tool
|
||||
|
||||
_scrapy() {
|
||||
local curcontext="$curcontext" cmd spiders
|
||||
local context state state_descr line
|
||||
typeset -A opt_args
|
||||
cmd=$words[2]
|
||||
|
||||
case "$cmd" in
|
||||
crawl|edit|check)
|
||||
spiders=$(scrapy list 2>/dev/null) || spiders=""
|
||||
if [[ -n "$spiders" ]]; then
|
||||
compadd `echo $spiders`
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if [[ CURRENT -eq 2 ]]; then
|
||||
_arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
|
||||
fi
|
||||
;;
|
||||
_arguments \
|
||||
"(- 1 *)--help[Help]" \
|
||||
"1: :->command" \
|
||||
"*:: :->args"
|
||||
|
||||
case $state in
|
||||
command)
|
||||
_scrapy_cmds
|
||||
;;
|
||||
args)
|
||||
case $words[1] in
|
||||
bench)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
fetch)
|
||||
local options=(
|
||||
'--headers[print response HTTP headers instead of body]'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'1::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
genspider)
|
||||
local options=(
|
||||
{-l,--list}'[List available templates]'
|
||||
{-e,--edit}'[Edit spider after creating it]'
|
||||
'--force[If the spider already exists, overwrite it with the template]'
|
||||
{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
|
||||
{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
|
||||
'1:name:(NAME)'
|
||||
'2:domain:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
runspider)
|
||||
local options=(
|
||||
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider file:_files -g \*.py'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
settings)
|
||||
local options=(
|
||||
'--get=[print raw setting value]:option:(SETTING)'
|
||||
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
|
||||
'--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
|
||||
'--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
|
||||
'--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
shell)
|
||||
local options=(
|
||||
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'::file:_files -g \*.html'
|
||||
'::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
startproject)
|
||||
local options=(
|
||||
'1:name:(NAME)'
|
||||
'2:dir:_dir_list'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
version)
|
||||
local options=(
|
||||
{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
view)
|
||||
local options=(
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
check)
|
||||
local options=(
|
||||
'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
|
||||
{-v,--verbose}'[print contract tests for all spiders]'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
crawl)
|
||||
local options=(
|
||||
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
edit)
|
||||
local options=(
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
list)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
parse)
|
||||
local options=(
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
|
||||
'--pipelines[process items through pipelines]'
|
||||
"--nolinks[don't show links to follow (extracted requests)]"
|
||||
"--noitems[don't show scraped items]"
|
||||
'--nocolour[avoid using pygments to colorize the output]'
|
||||
{-r,--rules}'[use CrawlSpider rules to discover the callback]'
|
||||
{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
|
||||
{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
|
||||
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
|
||||
{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
|
||||
{-v,--verbose}'[print each depth level one by one]'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
_scrapy
|
||||
_scrapy_cmds() {
|
||||
local -a commands project_commands
|
||||
commands=(
|
||||
'bench:Run quick benchmark test'
|
||||
'fetch:Fetch a URL using the Scrapy downloader'
|
||||
'genspider:Generate new spider using pre-defined templates'
|
||||
'runspider:Run a self-contained spider (without creating a project)'
|
||||
'settings:Get settings values'
|
||||
'shell:Interactive scraping console'
|
||||
'startproject:Create new project'
|
||||
'version:Print Scrapy version'
|
||||
'view:Open URL in browser, as seen by Scrapy'
|
||||
)
|
||||
project_commands=(
|
||||
'check:Check spider contracts'
|
||||
'crawl:Run a spider'
|
||||
'edit:Edit spider'
|
||||
'list:List available spiders'
|
||||
'parse:Parse URL (using its spider) and print the results'
|
||||
)
|
||||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
commands=(${commands[@]} ${project_commands[@]})
|
||||
fi
|
||||
_describe -t common-commands 'common commands' commands
|
||||
}
|
||||
|
||||
_scrapy_glb_opts() {
|
||||
local -a options
|
||||
options=(
|
||||
'(- *)'{-h,--help}'[show this help message and exit]'
|
||||
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
|
||||
'--pidfile=[write process ID to FILE]:file:_files'
|
||||
'--profile=[write python cProfile stats to FILE]:file:_files'
|
||||
'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
|
||||
'(-L --loglevel --logfile)--nolog[disable logging completely]'
|
||||
'--pdb[enable pdb on failure]'
|
||||
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
)
|
||||
options=(${options[@]} "$@")
|
||||
_arguments $options
|
||||
}
|
||||
|
||||
_httpie_urls() {
|
||||
|
||||
local ret=1
|
||||
|
||||
if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
|
||||
local expl
|
||||
compset -S '[^:/]*' && compstate[to_end]=''
|
||||
_wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
|
||||
else
|
||||
_urls && ret=0
|
||||
fi
|
||||
|
||||
return $ret
|
||||
|
||||
}
|
||||
|
||||
_scrapy_spiders() {
|
||||
|
||||
local ret=1
|
||||
|
||||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
compadd -S '' $(scrapy list) && ret=0
|
||||
else
|
||||
compadd -S '' SPIDER && ret=0
|
||||
fi
|
||||
|
||||
return $ret
|
||||
}
|
||||
|
||||
_scrapy $@
|
||||
|
|
|
|||
267
pytest.ini
267
pytest.ini
|
|
@ -2,5 +2,270 @@
|
|||
usefixtures = chdir
|
||||
python_files=test_*.py __init__.py
|
||||
python_classes=
|
||||
addopts = --doctest-modules --assert=plain
|
||||
addopts =
|
||||
--assert=plain
|
||||
--doctest-modules
|
||||
--ignore=docs/_ext
|
||||
--ignore=docs/conf.py
|
||||
--ignore=docs/news.rst
|
||||
--ignore=docs/topics/commands.rst
|
||||
--ignore=docs/topics/debug.rst
|
||||
--ignore=docs/topics/developer-tools.rst
|
||||
--ignore=docs/topics/dynamic-content.rst
|
||||
--ignore=docs/topics/items.rst
|
||||
--ignore=docs/topics/leaks.rst
|
||||
--ignore=docs/topics/loaders.rst
|
||||
--ignore=docs/topics/selectors.rst
|
||||
--ignore=docs/topics/shell.rst
|
||||
--ignore=docs/topics/stats.rst
|
||||
--ignore=docs/topics/telnetconsole.rst
|
||||
--ignore=docs/utils
|
||||
twisted = 1
|
||||
flake8-ignore =
|
||||
# extras
|
||||
extras/qps-bench-server.py E261 E501
|
||||
extras/qpsclient.py E501 E261 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py F401 E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/list.py E302
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E302 E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/startproject.py E502 E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
scrapy/commands/view.py F401 E302
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E502 E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E261 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E502 E126 E226
|
||||
scrapy/core/downloader/__init__.py F401 E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E502 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py F401 E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E502 E128 E123
|
||||
scrapy/extensions/corestats.py E302 E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303 F401
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E302 E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128 W293 W291
|
||||
scrapy/http/response/html.py E302
|
||||
scrapy/http/response/text.py E501 W293 E128 E124
|
||||
scrapy/http/response/xml.py E302
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E502 E128
|
||||
scrapy/loader/common.py E302
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/__init__.py E302
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
scrapy/pipelines/images.py E265 E501
|
||||
scrapy/pipelines/media.py E125 E501 E266
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403 F401
|
||||
scrapy/selector/unified.py F401 E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py F401 E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501 E261
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/boto.py F401
|
||||
scrapy/utils/conf.py E402 E502 E501
|
||||
scrapy/utils/console.py E302 E261 F401 E306 E305
|
||||
scrapy/utils/curl.py F401
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/decorators.py E501 E302
|
||||
scrapy/utils/defer.py E501 E302 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/display.py E302
|
||||
scrapy/utils/engine.py F401 E261 E302
|
||||
scrapy/utils/ftp.py E302
|
||||
scrapy/utils/gz.py E305 E501 E302 W504
|
||||
scrapy/utils/http.py F403 F401 E226
|
||||
scrapy/utils/httpobj.py E302 E501
|
||||
scrapy/utils/iterators.py E501 E701
|
||||
scrapy/utils/job.py E302
|
||||
scrapy/utils/log.py E128 W503
|
||||
scrapy/utils/markup.py F403 F401 W292
|
||||
scrapy/utils/misc.py E501 E226
|
||||
scrapy/utils/multipart.py F403 F401 W292
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501 E302
|
||||
scrapy/utils/reactor.py E302 E226
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E302 E127 E501
|
||||
scrapy/utils/response.py E501 E302 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E271 E302 E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/template.py E302
|
||||
scrapy/utils/test.py E302 E501
|
||||
scrapy/utils/url.py E501 F403 F401 E128 F405
|
||||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/_monkeypatches.py W293
|
||||
scrapy/cmdline.py E502 E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E302 E501 E202
|
||||
scrapy/exceptions.py E302 E501
|
||||
scrapy/exporters.py E501 E261 E226
|
||||
scrapy/extension.py E302
|
||||
scrapy/interfaces.py E302 E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501 W293
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/middleware.py E502 E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E302
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/robotstxt.py E302 E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py E225 F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py F401 E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123 F401
|
||||
tests/pipelines.py E302 F841 E226
|
||||
tests/spiders.py E302 E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501 E261
|
||||
tests/test_command_parse.py F401 E302 E501 E128 E303 E226
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py F401 E128 E501
|
||||
tests/test_contracts.py E501 E128 W293
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py E302 F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E502 E128 E261
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_extension_telnet.py F401 F841
|
||||
tests/test_feedexport.py E501 F401 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E302 E501
|
||||
tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_response.py E501 E301 E502 E128 E265
|
||||
tests/test_item.py E701 E128 F841 E306
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122 E302
|
||||
tests/test_mail.py E302 E128 E501 E305
|
||||
tests/test_middleware.py E302 E501 E128
|
||||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F401 F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E302 E305
|
||||
tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py F401 E501 E127
|
||||
tests/test_spider.py E501 F401
|
||||
tests/test_spidermiddleware.py E501 E226
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
|
||||
tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E302 E701 E741
|
||||
tests/test_utils_conf.py E501 E303 E128
|
||||
tests/test_utils_console.py E302
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_http.py E302 E501 E502 E128 W504
|
||||
tests/test_utils_httpobj.py E302
|
||||
tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
tests/test_utils_reqser.py F401 E501 E128
|
||||
tests/test_utils_request.py E302 E501 E128 E305
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E302 E731 E226
|
||||
tests/test_utils_sitemap.py E302 E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E302 E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/mocks/dummydbm.py E302
|
||||
tests/test_cmdline/__init__.py E502 E501
|
||||
tests/test_cmdline/extensions.py E302
|
||||
tests/test_settings/__init__.py F401 E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501 E302
|
||||
tests/test_spiderloader/test_spiders/spider0.py E302
|
||||
tests/test_spiderloader/test_spiders/spider1.py E302
|
||||
tests/test_spiderloader/test_spiders/spider2.py E302
|
||||
tests/test_spiderloader/test_spiders/spider3.py E302
|
||||
tests/test_spiderloader/test_spiders/nested/spider4.py E302
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
parsel>=1.5.0
|
||||
PyDispatcher>=2.0.5
|
||||
w3lib>=1.17.0
|
||||
protego>=0.1.15
|
||||
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
|
||||
# Reference versions taken from
|
||||
# https://packages.ubuntu.com/xenial/python/
|
||||
# https://packages.ubuntu.com/xenial/zope/
|
||||
cssselect>=0.9.1
|
||||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
Twisted>=16.0.0
|
||||
zope.interface>=4.1.3
|
||||
|
|
@ -1 +1 @@
|
|||
1.7.0
|
||||
1.8.0
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ del pkgutil
|
|||
|
||||
# Check minimum required Python version
|
||||
import sys
|
||||
if sys.version_info < (2, 7):
|
||||
print("Scrapy %s requires Python 2.7" % __version__)
|
||||
if sys.version_info < (3, 5):
|
||||
print("Scrapy %s requires Python 3.5" % __version__)
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore noisy twisted deprecation warnings
|
||||
|
|
|
|||
|
|
@ -1,16 +1,6 @@
|
|||
import six
|
||||
from six.moves import copyreg
|
||||
|
||||
|
||||
if six.PY2:
|
||||
from urlparse import urlparse
|
||||
|
||||
# workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
|
||||
if urlparse('s3://bucket/key?key=value').query != 'key=value':
|
||||
from urlparse import uses_query
|
||||
uses_query.append('s3')
|
||||
|
||||
|
||||
# Undo what Twisted's perspective broker adds to pickle register
|
||||
# to prevent bugs like Twisted#7989 while serializing requests
|
||||
import twisted.persisted.styles # NOQA
|
||||
|
|
|
|||
|
|
@ -96,4 +96,3 @@ class Command(ScrapyCommand):
|
|||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from __future__ import print_function
|
||||
import sys, six
|
||||
import sys
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -45,8 +45,7 @@ class Command(ScrapyCommand):
|
|||
self._print_bytes(response.body)
|
||||
|
||||
def _print_bytes(self, bytes_):
|
||||
bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
|
||||
bytes_writer.write(bytes_ + b'\n')
|
||||
sys.stdout.buffer.write(bytes_ + b'\n')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
|
|
|
|||
|
|
@ -119,4 +119,3 @@ class Command(ScrapyCommand):
|
|||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'project')
|
||||
|
||||
|
|
|
|||
|
|
@ -30,4 +30,3 @@ class Command(ScrapyCommand):
|
|||
print(patt % (name, version))
|
||||
else:
|
||||
print("Scrapy %s" % scrapy.__version__)
|
||||
|
||||
|
|
|
|||
|
|
@ -112,4 +112,3 @@ class FTPDownloadHandler(object):
|
|||
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
|
||||
return Response(url=request.url, status=httpcode, body=to_bytes(message))
|
||||
raise result.type(result.value)
|
||||
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
|
|||
r"""
|
||||
Return binary content of a CONNECT request.
|
||||
|
||||
>>> from scrapy.utils.python import to_native_str as s
|
||||
>>> from scrapy.utils.python import to_unicode as s
|
||||
>>> s(tunnel_request_data("example.com", 8080))
|
||||
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
|
||||
>>> s(tunnel_request_data("example.com", 8080, b"123"))
|
||||
|
|
|
|||
|
|
@ -157,4 +157,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
def gotHeaders(self, headers):
|
||||
self.headers_time = time()
|
||||
self.response_headers = headers
|
||||
|
||||
|
|
|
|||
|
|
@ -231,9 +231,9 @@ class Scraper(object):
|
|||
signal=signals.item_dropped, item=item, response=response,
|
||||
spider=spider, exception=output.value)
|
||||
else:
|
||||
logger.error('Error processing %(item)s', {'item': item},
|
||||
exc_info=failure_to_exc_info(output),
|
||||
extra={'spider': spider})
|
||||
logkws = self.logformatter.error(item, ex, response, spider)
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
|
||||
exc_info=failure_to_exc_info(output))
|
||||
return self.signals.send_catch_log_deferred(
|
||||
signal=signals.item_error, item=item, response=response,
|
||||
spider=spider, failure=output)
|
||||
|
|
@ -244,4 +244,3 @@ class Scraper(object):
|
|||
return self.signals.send_catch_log_deferred(
|
||||
signal=signals.item_scraped, item=output, response=response,
|
||||
spider=spider)
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
|
||||
|
||||
def scrape_response(self, scrape_func, response, request, spider):
|
||||
fname = lambda f:'%s.%s' % (
|
||||
fname = lambda f: '%s.%s' % (
|
||||
six.get_method_self(f).__class__.__name__,
|
||||
six.get_method_function(f).__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import signal
|
|||
import logging
|
||||
import warnings
|
||||
|
||||
import sys
|
||||
from twisted.internet import reactor, defer
|
||||
from zope.interface.verify import verifyClass, DoesNotImplement
|
||||
|
||||
|
|
@ -88,20 +87,9 @@ class Crawler(object):
|
|||
yield self.engine.open_spider(self.spider, start_requests)
|
||||
yield defer.maybeDeferred(self.engine.start)
|
||||
except Exception:
|
||||
# In Python 2 reraising an exception after yield discards
|
||||
# the original traceback (see https://bugs.python.org/issue7563),
|
||||
# so sys.exc_info() workaround is used.
|
||||
# This workaround also works in Python 3, but it is not needed,
|
||||
# and it is slower, so in Python 3 we use native `raise`.
|
||||
if six.PY2:
|
||||
exc_info = sys.exc_info()
|
||||
|
||||
self.crawling = False
|
||||
if self.engine is not None:
|
||||
yield self.engine.close()
|
||||
|
||||
if six.PY2:
|
||||
six.reraise(*exc_info)
|
||||
raise
|
||||
|
||||
def _create_spider(self, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from collections import defaultdict
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -53,7 +53,7 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _debug_cookie(self, request, spider):
|
||||
if self.debug:
|
||||
cl = [to_native_str(c, errors='replace')
|
||||
cl = [to_unicode(c, errors='replace')
|
||||
for c in request.headers.getlist('Cookie')]
|
||||
if cl:
|
||||
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
|
||||
|
|
@ -62,7 +62,7 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _debug_set_cookie(self, response, spider):
|
||||
if self.debug:
|
||||
cl = [to_native_str(c, errors='replace')
|
||||
cl = [to_unicode(c, errors='replace')
|
||||
for c in response.headers.getlist('Set-Cookie')]
|
||||
if cl:
|
||||
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive.
|
|||
|
||||
import bz2
|
||||
import gzip
|
||||
from io import BytesIO
|
||||
import zipfile
|
||||
import tarfile
|
||||
import logging
|
||||
|
|
@ -11,11 +12,6 @@ from tempfile import mktemp
|
|||
|
||||
import six
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
import base64
|
||||
from six.moves.urllib.parse import unquote, urlunparse
|
||||
from six.moves.urllib.request import getproxies, proxy_bypass
|
||||
try:
|
||||
from urllib2 import _parse_proxy
|
||||
except ImportError:
|
||||
from urllib.request import _parse_proxy
|
||||
from urllib.request import _parse_proxy
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from six.moves.urllib.parse import urljoin, urlparse
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
|
|
@ -70,7 +70,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
if 'Location' not in response.headers or response.status not in allowed_status:
|
||||
return response
|
||||
|
||||
location = safe_url_string(response.headers['location'])
|
||||
location = safe_url_string(response.headers['Location'])
|
||||
if response.headers['Location'].startswith(b'//'):
|
||||
request_scheme = urlparse(request.url).scheme
|
||||
location = request_scheme + '://' + location.lstrip('/')
|
||||
|
||||
redirected_url = urljoin(request.url, location)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
|
|||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import re
|
||||
|
||||
from twisted.internet.defer import Deferred, maybeDeferred
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
import pprint
|
||||
import marshal
|
||||
import six
|
||||
|
|
@ -12,7 +11,7 @@ from six.moves import cPickle as pickle
|
|||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
|
||||
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
import warnings
|
||||
|
|
@ -31,7 +30,7 @@ class BaseItemExporter(object):
|
|||
def _configure(self, options, dont_fail=False):
|
||||
"""Configure the exporter by poping options from the ``options`` dict.
|
||||
If dont_fail is set, it won't raise an exception on unexpected options
|
||||
(useful for using with keyword arguments in subclasses constructors)
|
||||
(useful for using with keyword arguments in subclasses ``__init__`` methods)
|
||||
"""
|
||||
self.encoding = options.pop('encoding', None)
|
||||
self.fields_to_export = options.pop('fields_to_export', None)
|
||||
|
|
@ -143,11 +142,11 @@ class XmlItemExporter(BaseItemExporter):
|
|||
|
||||
def _beautify_newline(self, new_item=False):
|
||||
if self.indent is not None and (self.indent > 0 or new_item):
|
||||
self._xg_characters('\n')
|
||||
self.xg.characters('\n')
|
||||
|
||||
def _beautify_indent(self, depth=1):
|
||||
if self.indent:
|
||||
self._xg_characters(' ' * self.indent * depth)
|
||||
self.xg.characters(' ' * self.indent * depth)
|
||||
|
||||
def start_exporting(self):
|
||||
self.xg.startDocument()
|
||||
|
|
@ -182,26 +181,12 @@ class XmlItemExporter(BaseItemExporter):
|
|||
self._export_xml_field('value', value, depth=depth+1)
|
||||
self._beautify_indent(depth=depth)
|
||||
elif isinstance(serialized_value, six.text_type):
|
||||
self._xg_characters(serialized_value)
|
||||
self.xg.characters(serialized_value)
|
||||
else:
|
||||
self._xg_characters(str(serialized_value))
|
||||
self.xg.characters(str(serialized_value))
|
||||
self.xg.endElement(name)
|
||||
self._beautify_newline()
|
||||
|
||||
# Workaround for https://bugs.python.org/issue17606
|
||||
# Before Python 2.7.4 xml.sax.saxutils required bytes;
|
||||
# since 2.7.4 it requires unicode. The bug is likely to be
|
||||
# fixed in 2.7.6, but 2.7.6 will still support unicode,
|
||||
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
|
||||
if sys.version_info[:3] >= (2, 7, 4):
|
||||
def _xg_characters(self, serialized_value):
|
||||
if not isinstance(serialized_value, six.text_type):
|
||||
serialized_value = serialized_value.decode(self.encoding)
|
||||
return self.xg.characters(serialized_value)
|
||||
else: # pragma: no cover
|
||||
def _xg_characters(self, serialized_value):
|
||||
return self.xg.characters(serialized_value)
|
||||
|
||||
|
||||
class CsvItemExporter(BaseItemExporter):
|
||||
|
||||
|
|
@ -216,7 +201,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
write_through=True,
|
||||
encoding=self.encoding,
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
) if six.PY3 else file
|
||||
)
|
||||
self.csv_writer = csv.writer(self.stream, **kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
|
|
@ -246,7 +231,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def _build_row(self, values):
|
||||
for s in values:
|
||||
try:
|
||||
yield to_native_str(s, self.encoding)
|
||||
yield to_unicode(s, self.encoding)
|
||||
except TypeError:
|
||||
yield s
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import logging
|
|||
import posixpath
|
||||
from tempfile import NamedTemporaryFile
|
||||
from datetime import datetime
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse, unquote
|
||||
from ftplib import FTP
|
||||
|
||||
|
|
@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
|
|||
|
||||
def __init__(self, uri, _stdout=None):
|
||||
if not _stdout:
|
||||
_stdout = sys.stdout if six.PY2 else sys.stdout.buffer
|
||||
_stdout = sys.stdout.buffer
|
||||
self._stdout = _stdout
|
||||
|
||||
def open(self, spider):
|
||||
|
|
@ -199,9 +198,9 @@ class FeedExporter(object):
|
|||
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.urifmt = settings['FEED_URI']
|
||||
if not self.urifmt:
|
||||
if not settings['FEED_URI']:
|
||||
raise NotConfigured
|
||||
self.urifmt = str(settings['FEED_URI'])
|
||||
self.format = settings['FEED_FORMAT'].lower()
|
||||
self.export_encoding = settings['FEED_EXPORT_ENCODING']
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
|
|
@ -242,7 +241,9 @@ class FeedExporter(object):
|
|||
def close_spider(self, spider):
|
||||
slot = self.slot
|
||||
if not slot.itemcount and not self.store_empty:
|
||||
return
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
if self._exporting:
|
||||
slot.exporter.finish_exporting()
|
||||
self._exporting = False
|
||||
|
|
|
|||
|
|
@ -1,19 +1,24 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
from six.moves import cPickle as pickle
|
||||
import os
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from importlib import import_module
|
||||
from time import time
|
||||
from warnings import warn
|
||||
from weakref import WeakKeyDictionary
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
|
||||
from six.moves import cPickle as pickle
|
||||
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -342,75 +347,6 @@ class FilesystemCacheStorage(object):
|
|||
return pickle.load(f)
|
||||
|
||||
|
||||
class LeveldbCacheStorage(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
import leveldb
|
||||
self._leveldb = leveldb
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
self.db = None
|
||||
|
||||
def open_spider(self, spider):
|
||||
dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
|
||||
self.db = self._leveldb.LevelDB(dbpath)
|
||||
|
||||
logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
def close_spider(self, spider):
|
||||
# Do compactation each time to save space and also recreate files to
|
||||
# avoid them being removed in storages with timestamp-based autoremoval.
|
||||
self.db.CompactRange()
|
||||
del self.db
|
||||
garbage_collect()
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
url = data['url']
|
||||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
body = data['body']
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
'headers': dict(response.headers),
|
||||
'body': response.body,
|
||||
}
|
||||
batch = self._leveldb.WriteBatch()
|
||||
batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
|
||||
batch.Put(key + b'_time', to_bytes(str(time())))
|
||||
self.db.Write(batch)
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
try:
|
||||
ts = self.db.Get(key + b'_time')
|
||||
except KeyError:
|
||||
return # not found or invalid entry
|
||||
|
||||
if 0 < self.expiration_secs < time() - float(ts):
|
||||
return # expired
|
||||
|
||||
try:
|
||||
data = self.db.Get(key + b'_data')
|
||||
except KeyError:
|
||||
return # invalid entry
|
||||
else:
|
||||
return pickle.loads(data)
|
||||
|
||||
def _request_key(self, request):
|
||||
return to_bytes(request_fingerprint(request))
|
||||
|
||||
|
||||
|
||||
def parse_cachecontrol(header):
|
||||
"""Parse Cache-Control header
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
|
|||
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
|
||||
)
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
class CookieJar(object):
|
||||
|
|
@ -165,13 +165,13 @@ class WrappedRequest(object):
|
|||
return name in self.request.headers
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return to_native_str(self.request.headers.get(name, default),
|
||||
errors='replace')
|
||||
return to_unicode(self.request.headers.get(name, default),
|
||||
errors='replace')
|
||||
|
||||
def header_items(self):
|
||||
return [
|
||||
(to_native_str(k, errors='replace'),
|
||||
[to_native_str(x, errors='replace') for x in v])
|
||||
(to_unicode(k, errors='replace'),
|
||||
[to_unicode(x, errors='replace') for x in v])
|
||||
for k, v in self.request.headers.items()
|
||||
]
|
||||
|
||||
|
|
@ -189,7 +189,7 @@ class WrappedResponse(object):
|
|||
|
||||
# python3 cookiejars calls get_all
|
||||
def get_all(self, name, default=None):
|
||||
return [to_native_str(v, errors='replace')
|
||||
return [to_unicode(v, errors='replace')
|
||||
for v in self.response.headers.getlist(name)]
|
||||
# python2 cookiejars calls getheaders
|
||||
getheaders = get_all
|
||||
|
|
|
|||
|
|
@ -91,5 +91,3 @@ class Headers(CaselessDict):
|
|||
def __copy__(self):
|
||||
return self.__class__(self)
|
||||
copy = __copy__
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ class Request(object_ref):
|
|||
s = safe_url_string(url, self.encoding)
|
||||
self._url = escape_ajax(s)
|
||||
|
||||
if ':' not in self._url:
|
||||
if ('://' not in self._url) and (not self._url.startswith('data:')):
|
||||
raise ValueError('Missing scheme in request url: %s' % self._url)
|
||||
|
||||
url = property(_get_url, obsolete_setter(_set_url, 'url'))
|
||||
|
|
|
|||
|
|
@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
|
|||
el = el.getparent()
|
||||
if el is None:
|
||||
break
|
||||
encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
|
||||
raise ValueError('No <form> element found with %s' % encoded)
|
||||
raise ValueError('No <form> element found with %s' % formxpath)
|
||||
|
||||
# If we get here, it means that either formname was None
|
||||
# or invalid
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ class Response(object_ref):
|
|||
@property
|
||||
def text(self):
|
||||
"""For subclasses of TextResponse, this will return the body
|
||||
as text (unicode object in Python 2 and str in Python 3)
|
||||
as str
|
||||
"""
|
||||
raise AttributeError("Response content isn't text")
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
|
|||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_native_str
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
from scrapy.utils.response import get_base_url
|
||||
|
||||
|
||||
|
|
@ -32,10 +32,7 @@ class TextResponse(Response):
|
|||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, six.text_type):
|
||||
if six.PY2 and self.encoding is None:
|
||||
raise TypeError("Cannot convert unicode url - %s "
|
||||
"has no encoding" % type(self).__name__)
|
||||
self._url = to_native_str(url, self.encoding)
|
||||
self._url = to_unicode(url, self.encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_url(url)
|
||||
|
||||
|
|
@ -84,11 +81,11 @@ class TextResponse(Response):
|
|||
@memoizemethod_noargs
|
||||
def _headers_encoding(self):
|
||||
content_type = self.headers.get(b'Content-Type', b'')
|
||||
return http_content_type_encoding(to_native_str(content_type))
|
||||
return http_content_type_encoding(to_unicode(content_type))
|
||||
|
||||
def _body_inferred_encoding(self):
|
||||
if self._cached_benc is None:
|
||||
content_type = to_native_str(self.headers.get(b'Content-Type', b''))
|
||||
content_type = to_unicode(self.headers.get(b'Content-Type', b''))
|
||||
benc, ubody = html_to_unicode(content_type, self.body,
|
||||
auto_detect_fun=self._auto_detect_fun,
|
||||
default_encoding=self._DEFAULT_ENCODING)
|
||||
|
|
|
|||
|
|
@ -15,4 +15,3 @@ class ISpiderLoader(Interface):
|
|||
|
||||
def find_by_request(request):
|
||||
"""Return the list of spiders names that can handle the given request"""
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ Scrapy Item
|
|||
See documentation in docs/topics/item.rst
|
||||
"""
|
||||
|
||||
import collections
|
||||
from abc import ABCMeta
|
||||
from collections.abc import MutableMapping
|
||||
from copy import deepcopy
|
||||
from pprint import pformat
|
||||
from warnings import warn
|
||||
|
|
@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
|||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
if six.PY2:
|
||||
MutableMapping = collections.MutableMapping
|
||||
else:
|
||||
MutableMapping = collections.abc.MutableMapping
|
||||
|
||||
|
||||
class BaseItem(object_ref):
|
||||
"""Base class for all scraped items.
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,6 @@ This module defines the Link object used in Link extractors.
|
|||
For actual link extractors implementation see scrapy.linkextractors, or
|
||||
its documentation in: docs/topics/link-extractors.rst
|
||||
"""
|
||||
import warnings
|
||||
import six
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
||||
class Link(object):
|
||||
|
|
@ -17,13 +13,8 @@ class Link(object):
|
|||
|
||||
def __init__(self, url, text='', fragment='', nofollow=False):
|
||||
if not isinstance(url, str):
|
||||
if six.PY2:
|
||||
warnings.warn("Link urls must be str objects. "
|
||||
"Assuming utf-8 encoding (which could be wrong)")
|
||||
url = to_bytes(url, encoding='utf8')
|
||||
else:
|
||||
got = url.__class__.__name__
|
||||
raise TypeError("Link urls must be str objects, got %s" % got)
|
||||
got = url.__class__.__name__
|
||||
raise TypeError("Link urls must be str objects, got %s" % got)
|
||||
self.url = url
|
||||
self.text = text
|
||||
self.fragment = fragment
|
||||
|
|
@ -39,4 +30,3 @@ class Link(object):
|
|||
def __repr__(self):
|
||||
return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
|
||||
(self.url, self.text, self.fragment, self.nofollow)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,23 +19,26 @@ from scrapy.utils.url import (
|
|||
|
||||
# common file extensions that are not followed if they occur in links
|
||||
IGNORED_EXTENSIONS = [
|
||||
# archives
|
||||
'7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip',
|
||||
|
||||
# images
|
||||
'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
|
||||
'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg',
|
||||
'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico',
|
||||
|
||||
# audio
|
||||
'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff',
|
||||
|
||||
# video
|
||||
'3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
|
||||
'm4a', 'm4v', 'flv',
|
||||
'm4a', 'm4v', 'flv', 'webm',
|
||||
|
||||
# office suites
|
||||
'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',
|
||||
'odp',
|
||||
|
||||
# other
|
||||
'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar',
|
||||
'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk'
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
|
|||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list, to_native_str
|
||||
from scrapy.utils.python import unique as unique_list, to_unicode
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
|
|||
url = self.process_attr(attr_val)
|
||||
if url is None:
|
||||
continue
|
||||
url = to_native_str(url, encoding=response_encoding)
|
||||
url = to_unicode(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
"""Item Loader
|
||||
"""
|
||||
Item Loader
|
||||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
|
||||
"""
|
||||
from collections import defaultdict
|
||||
|
||||
import six
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.loader.processors import Identity
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.utils.misc import arg_to_iter, extract_regex
|
||||
from scrapy.utils.python import flatten
|
||||
|
||||
from .common import wrap_loader_context
|
||||
from .processors import Identity
|
||||
|
||||
|
||||
class ItemLoader(object):
|
||||
|
||||
|
|
@ -33,10 +33,9 @@ class ItemLoader(object):
|
|||
self.parent = parent
|
||||
self._local_item = context['item'] = item
|
||||
self._local_values = defaultdict(list)
|
||||
# Preprocess values if item built from dict
|
||||
# Values need to be added to item._values if added them from dict (not with add_values)
|
||||
# values from initial item
|
||||
for field_name, value in item.items():
|
||||
self._values[field_name] = self._process_input_value(field_name, value)
|
||||
self._values[field_name] += arg_to_iter(value)
|
||||
|
||||
@property
|
||||
def _values(self):
|
||||
|
|
@ -132,8 +131,8 @@ class ItemLoader(object):
|
|||
try:
|
||||
return proc(self._values[field_name])
|
||||
except Exception as e:
|
||||
raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
|
||||
(field_name, self._values[field_name], type(e).__name__, str(e)))
|
||||
raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
|
||||
(field_name, self._values[field_name], type(e).__name__, str(e)))
|
||||
|
||||
def get_collected_values(self, field_name):
|
||||
return self._values[field_name]
|
||||
|
|
@ -141,15 +140,15 @@ class ItemLoader(object):
|
|||
def get_input_processor(self, field_name):
|
||||
proc = getattr(self, '%s_in' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor', \
|
||||
self.default_input_processor)
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor',
|
||||
self.default_input_processor)
|
||||
return proc
|
||||
|
||||
def get_output_processor(self, field_name):
|
||||
proc = getattr(self, '%s_out' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor', \
|
||||
self.default_output_processor)
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor',
|
||||
self.default_output_processor)
|
||||
return proc
|
||||
|
||||
def _process_input_value(self, field_name, value):
|
||||
|
|
@ -174,8 +173,8 @@ class ItemLoader(object):
|
|||
def _check_selector_method(self):
|
||||
if self.selector is None:
|
||||
raise RuntimeError("To use XPath or CSS selectors, "
|
||||
"%s must be instantiated with a selector "
|
||||
"or a response" % self.__class__.__name__)
|
||||
"%s must be instantiated with a selector "
|
||||
"or a response" % self.__class__.__name__)
|
||||
|
||||
def add_xpath(self, field_name, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
|
|
|
|||
|
|
@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
try:
|
||||
from collections import ChainMap
|
||||
except ImportError:
|
||||
from scrapy.utils.datatypes import MergeDict as ChainMap
|
||||
from collections import ChainMap
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from scrapy.utils.request import referer_str
|
|||
SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
|
||||
DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
|
||||
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
|
||||
ERRORMSG = u"'Error processing %(item)s'"
|
||||
|
||||
|
||||
class LogFormatter(object):
|
||||
|
|
@ -29,7 +30,7 @@ class LogFormatter(object):
|
|||
* ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
|
||||
The final log message is computed as ``msg % args``.
|
||||
|
||||
Users can define their own ``LogFormatter`` class if they want to customise how
|
||||
Users can define their own ``LogFormatter`` class if they want to customize how
|
||||
each action is logged or if they want to omit it entirely. In order to omit
|
||||
logging an action the method must return ``None``.
|
||||
|
||||
|
|
@ -92,6 +93,16 @@ class LogFormatter(object):
|
|||
}
|
||||
}
|
||||
|
||||
def error(self, item, exception, response, spider):
|
||||
"""Logs a message when an item causes an error while it is passing through the item pipeline."""
|
||||
return {
|
||||
'level': logging.ERROR,
|
||||
'msg': ERRORMSG,
|
||||
'args': {
|
||||
'item': item,
|
||||
}
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls()
|
||||
|
|
|
|||
|
|
@ -3,24 +3,15 @@ Mail sending helpers
|
|||
|
||||
See documentation in docs/topics/email.rst
|
||||
"""
|
||||
from io import BytesIO
|
||||
import logging
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
import six
|
||||
|
||||
from email.utils import COMMASPACE, formatdate
|
||||
from six.moves.email_mime_multipart import MIMEMultipart
|
||||
from six.moves.email_mime_text import MIMEText
|
||||
from six.moves.email_mime_base import MIMEBase
|
||||
if six.PY2:
|
||||
from email.MIMENonMultipart import MIMENonMultipart
|
||||
from email import Encoders
|
||||
else:
|
||||
from email.mime.nonmultipart import MIMENonMultipart
|
||||
from email import encoders as Encoders
|
||||
from email.mime.nonmultipart import MIMENonMultipart
|
||||
from email import encoders as Encoders
|
||||
|
||||
from twisted.internet import defer, reactor, ssl
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
|
|||
"""
|
||||
import functools
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import mimetypes
|
||||
import os
|
||||
import os.path
|
||||
|
|
@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse
|
|||
from collections import defaultdict
|
||||
import six
|
||||
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from scrapy.pipelines.media import MediaPipeline
|
||||
|
|
|
|||
|
|
@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst
|
|||
"""
|
||||
import functools
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import six
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scrapy.utils.misc import md5sum
|
||||
|
|
|
|||
|
|
@ -86,9 +86,6 @@ class _SlotPriorityQueues(object):
|
|||
def __len__(self):
|
||||
return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
|
||||
|
||||
def __contains__(self, slot):
|
||||
return slot in self.pqueues
|
||||
|
||||
|
||||
class ScrapyPriorityQueue(PriorityQueue):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import six
|
|||
|
||||
from scrapy.http import Response
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
|
||||
from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
|
||||
|
||||
|
||||
class ResponseTypes(object):
|
||||
|
|
@ -55,12 +55,12 @@ class ResponseTypes(object):
|
|||
header """
|
||||
if content_encoding:
|
||||
return Response
|
||||
mimetype = to_native_str(content_type).split(';')[0].strip().lower()
|
||||
mimetype = to_unicode(content_type).split(';')[0].strip().lower()
|
||||
return self.from_mimetype(mimetype)
|
||||
|
||||
def from_content_disposition(self, content_disposition):
|
||||
try:
|
||||
filename = to_native_str(content_disposition,
|
||||
filename = to_unicode(content_disposition,
|
||||
encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
|
||||
filename = filename.strip('"\'')
|
||||
return self.from_filename(filename)
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@ import logging
|
|||
from abc import ABCMeta, abstractmethod
|
||||
from six import with_metaclass
|
||||
|
||||
from scrapy.utils.python import to_native_str, to_unicode
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
|
||||
try:
|
||||
if to_native_str_type:
|
||||
robotstxt_body = to_native_str(robotstxt_body)
|
||||
robotstxt_body = to_unicode(robotstxt_body)
|
||||
else:
|
||||
robotstxt_body = robotstxt_body.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
|
|
@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
|
|||
return o
|
||||
|
||||
def allowed(self, url, user_agent):
|
||||
user_agent = to_native_str(user_agent)
|
||||
url = to_native_str(url)
|
||||
user_agent = to_unicode(user_agent)
|
||||
url = to_unicode(url)
|
||||
return self.rp.can_fetch(user_agent, url)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,13 @@
|
|||
import six
|
||||
import json
|
||||
import copy
|
||||
import collections
|
||||
from collections.abc import MutableMapping
|
||||
from importlib import import_module
|
||||
from pprint import pformat
|
||||
|
||||
from scrapy.settings import default_settings
|
||||
|
||||
|
||||
if six.PY2:
|
||||
MutableMapping = collections.MutableMapping
|
||||
else:
|
||||
MutableMapping = collections.abc.MutableMapping
|
||||
|
||||
|
||||
SETTINGS_PRIORITIES = {
|
||||
'default': 0,
|
||||
'command': 10,
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ import sys
|
|||
from importlib import import_module
|
||||
from os.path import join, abspath, dirname
|
||||
|
||||
import six
|
||||
|
||||
AJAXCRAWL_ENABLED = False
|
||||
|
||||
AUTOTHROTTLE_ENABLED = False
|
||||
|
|
@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False
|
|||
HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
HTTPCACHE_IGNORE_SCHEMES = ['file']
|
||||
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
|
||||
HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
|
||||
HTTPCACHE_DBM_MODULE = 'dbm'
|
||||
HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
|
||||
HTTPCACHE_GZIP = False
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
|
|||
from scrapy.http import Request, Response
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy import signals
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.url import strip_url
|
||||
|
||||
|
|
@ -322,7 +321,7 @@ class RefererMiddleware(object):
|
|||
if isinstance(resp_or_url, Response):
|
||||
policy_header = resp_or_url.headers.get('Referrer-Policy')
|
||||
if policy_header is not None:
|
||||
policy_name = to_native_str(policy_header.decode('latin1'))
|
||||
policy_name = to_unicode(policy_header.decode('latin1'))
|
||||
if policy_name is None:
|
||||
return self.default_policy()
|
||||
|
||||
|
|
|
|||
|
|
@ -133,4 +133,3 @@ class CSVFeedSpider(Spider):
|
|||
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
|
||||
response = self.adapt_response(response)
|
||||
return self.parse_rows(response)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,4 +29,3 @@ class InitSpider(Spider):
|
|||
spider
|
||||
"""
|
||||
return self.initialized()
|
||||
|
||||
|
|
|
|||
|
|
@ -80,5 +80,3 @@ class DummyStatsCollector(StatsCollector):
|
|||
|
||||
def min_value(self, key, value, spider=None):
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Boto/botocore helpers"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
import six
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
|
|
@ -11,11 +10,4 @@ def is_botocore():
|
|||
import botocore
|
||||
return True
|
||||
except ImportError:
|
||||
if six.PY2:
|
||||
try:
|
||||
import boto
|
||||
return False
|
||||
except ImportError:
|
||||
raise NotConfigured('missing botocore or boto library')
|
||||
else:
|
||||
raise NotConfigured('missing botocore library')
|
||||
raise NotConfigured('missing botocore library')
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
from configparser import ConfigParser
|
||||
import os
|
||||
import sys
|
||||
import numbers
|
||||
from operator import itemgetter
|
||||
|
||||
import six
|
||||
if six.PY2:
|
||||
from ConfigParser import SafeConfigParser as ConfigParser
|
||||
else:
|
||||
from configparser import ConfigParser
|
||||
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.utils.deprecate import update_classpath
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library.
|
|||
|
||||
import copy
|
||||
import collections
|
||||
from collections.abc import Mapping
|
||||
import warnings
|
||||
|
||||
import six
|
||||
|
|
@ -14,12 +15,6 @@ import six
|
|||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
if six.PY2:
|
||||
Mapping = collections.Mapping
|
||||
else:
|
||||
Mapping = collections.abc.Mapping
|
||||
|
||||
|
||||
class MultiValueDictKeyError(KeyError):
|
||||
def __init__(self, *args, **kwargs):
|
||||
warnings.warn(
|
||||
|
|
@ -246,19 +241,18 @@ class CaselessDict(dict):
|
|||
class MergeDict(object):
|
||||
"""
|
||||
A simple class for creating new "virtual" dictionaries that actually look
|
||||
up values in more than one dictionary, passed in the constructor.
|
||||
up values in more than one dictionary, passed in the ``__init__`` method.
|
||||
|
||||
If a key appears in more than one of the given dictionaries, only the
|
||||
first occurrence will be used.
|
||||
"""
|
||||
def __init__(self, *dicts):
|
||||
if not six.PY2:
|
||||
warnings.warn(
|
||||
"scrapy.utils.datatypes.MergeDict is deprecated in favor "
|
||||
"of collections.ChainMap (introduced in Python 3.3)",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
warnings.warn(
|
||||
"scrapy.utils.datatypes.MergeDict is deprecated in favor "
|
||||
"of collections.ChainMap (introduced in Python 3.3)",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.dicts = dicts
|
||||
|
||||
def __getitem__(self, key):
|
||||
|
|
@ -315,8 +309,9 @@ class LocalCache(collections.OrderedDict):
|
|||
self.limit = limit
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
if self.limit:
|
||||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
super(LocalCache, self).__setitem__(key, value)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,7 @@
|
|||
import struct
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
from gzip import GzipFile
|
||||
|
||||
import six
|
||||
from io import BytesIO
|
||||
import re
|
||||
import struct
|
||||
|
||||
from scrapy.utils.decorators import deprecated
|
||||
|
||||
|
|
@ -17,14 +11,9 @@ from scrapy.utils.decorators import deprecated
|
|||
# (regression or bug-fix compared to Python 3.4)
|
||||
# - read1(), which fetches data before raising EOFError on next call
|
||||
# works here but is only available from Python>=3.3
|
||||
# - scrapy does not support Python 3.2
|
||||
# - Python 2.7 GzipFile works fine with standard read() + extrabuf
|
||||
if six.PY2:
|
||||
def read1(gzf, size=-1):
|
||||
return gzf.read(size)
|
||||
else:
|
||||
def read1(gzf, size=-1):
|
||||
return gzf.read1(size)
|
||||
@deprecated('GzipFile.read1')
|
||||
def read1(gzf, size=-1):
|
||||
return gzf.read1(size)
|
||||
|
||||
|
||||
def gunzip(data):
|
||||
|
|
@ -37,7 +26,7 @@ def gunzip(data):
|
|||
chunk = b'.'
|
||||
while chunk:
|
||||
try:
|
||||
chunk = read1(f, 8196)
|
||||
chunk = f.read1(8196)
|
||||
output_list.append(chunk)
|
||||
except (IOError, EOFError, struct.error):
|
||||
# complete only if there is some data, otherwise re-raise
|
||||
|
|
|
|||
|
|
@ -34,4 +34,3 @@ def decode_chunked_transfer(chunked_body):
|
|||
body += t[:size]
|
||||
t = t[size+2:]
|
||||
return body
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,7 @@
|
|||
import re
|
||||
import csv
|
||||
import logging
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
from io import StringIO
|
||||
import logging
|
||||
import six
|
||||
|
||||
from scrapy.http import TextResponse, Response
|
||||
|
|
@ -102,11 +98,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
|
|||
def row_to_unicode(row_):
|
||||
return [to_unicode(field, encoding) for field in row_]
|
||||
|
||||
# Python 3 csv reader input object needs to return strings
|
||||
if six.PY3:
|
||||
lines = StringIO(_body_or_str(obj, unicode=True))
|
||||
else:
|
||||
lines = BytesIO(_body_or_str(obj, unicode=False))
|
||||
lines = StringIO(_body_or_str(obj, unicode=True))
|
||||
|
||||
kwargs = {}
|
||||
if delimiter: kwargs["delimiter"] = delimiter
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
"""
|
||||
if settings is None:
|
||||
if crawler is None:
|
||||
raise ValueError("Specifiy at least one of settings and crawler.")
|
||||
raise ValueError("Specify at least one of settings and crawler.")
|
||||
settings = crawler.settings
|
||||
if crawler and hasattr(objcls, 'from_crawler'):
|
||||
return objcls.from_crawler(crawler, *args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'):
|
|||
if isinstance(text, six.text_type):
|
||||
return text
|
||||
if not isinstance(text, (bytes, six.text_type)):
|
||||
raise TypeError('to_unicode must receive a bytes, str or unicode '
|
||||
raise TypeError('to_unicode must receive a bytes or str '
|
||||
'object, got %s' % type(text).__name__)
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
|
|
@ -103,20 +103,17 @@ def to_bytes(text, encoding=None, errors='strict'):
|
|||
if isinstance(text, bytes):
|
||||
return text
|
||||
if not isinstance(text, six.string_types):
|
||||
raise TypeError('to_bytes must receive a unicode, str or bytes '
|
||||
raise TypeError('to_bytes must receive a str or bytes '
|
||||
'object, got %s' % type(text).__name__)
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
return text.encode(encoding, errors)
|
||||
|
||||
|
||||
@deprecated('to_unicode')
|
||||
def to_native_str(text, encoding=None, errors='strict'):
|
||||
""" Return str representation of ``text``
|
||||
(bytes in Python 2.x and unicode in Python 3.x). """
|
||||
if six.PY2:
|
||||
return to_bytes(text, encoding, errors)
|
||||
else:
|
||||
return to_unicode(text, encoding, errors)
|
||||
""" Return str representation of ``text``. """
|
||||
return to_unicode(text, encoding, errors)
|
||||
|
||||
|
||||
def re_rsearch(pattern, text, chunk_size=1024):
|
||||
|
|
@ -189,7 +186,7 @@ def _getargspec_py23(func):
|
|||
"""_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
|
||||
defaults)
|
||||
|
||||
Identical to inspect.getargspec() in python2, but uses
|
||||
Was identical to inspect.getargspec() in python2, but uses
|
||||
inspect.getfullargspec() for python3 behind the scenes to avoid
|
||||
DeprecationWarning.
|
||||
|
||||
|
|
@ -199,9 +196,6 @@ def _getargspec_py23(func):
|
|||
>>> _getargspec_py23(f)
|
||||
ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
|
||||
"""
|
||||
if six.PY2:
|
||||
return inspect.getargspec(func)
|
||||
|
||||
return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
|
||||
|
||||
|
||||
|
|
@ -303,7 +297,7 @@ class WeakKeyCache(object):
|
|||
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
|
||||
"""Return a (new) dict with unicode keys (and values when "keys_only" is
|
||||
False) of the given dict converted to strings. ``dct_or_tuples`` can be a
|
||||
dict or a list of tuples, like any dict constructor supports.
|
||||
dict or a list of tuples, like any dict ``__init__`` method supports.
|
||||
"""
|
||||
d = {}
|
||||
for k, v in six.iteritems(dict(dct_or_tuples)):
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ from twisted.internet import reactor, error
|
|||
def listen_tcp(portrange, host, factory):
|
||||
"""Like reactor.listenTCP but tries different ports in a range."""
|
||||
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
|
||||
if not hasattr(portrange, '__iter__'):
|
||||
return reactor.listenTCP(portrange, factory, interface=host)
|
||||
if not portrange:
|
||||
return reactor.listenTCP(0, factory, interface=host)
|
||||
if not hasattr(portrange, '__iter__'):
|
||||
return reactor.listenTCP(portrange, factory, interface=host)
|
||||
if len(portrange) == 1:
|
||||
return reactor.listenTCP(portrange[0], factory, interface=host)
|
||||
for x in range(portrange[0], portrange[1]+1):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
|
|||
import six
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.python import to_unicode, to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
|
|
@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
|
|||
eb = _get_method(spider, eb)
|
||||
request_cls = load_object(d['_class']) if '_class' in d else Request
|
||||
return request_cls(
|
||||
url=to_native_str(d['url']),
|
||||
url=to_unicode(d['url']),
|
||||
callback=cb,
|
||||
errback=eb,
|
||||
method=d['method'],
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@ import weakref
|
|||
from six.moves.urllib.parse import urlunparse
|
||||
|
||||
from w3lib.http import basic_auth_header
|
||||
from scrapy.utils.python import to_bytes, to_native_str
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
||||
from w3lib.url import canonicalize_url
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
|
||||
_fingerprint_cache = weakref.WeakKeyDictionary()
|
||||
def request_fingerprint(request, include_headers=None):
|
||||
def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
||||
"""
|
||||
Return the request fingerprint.
|
||||
|
||||
|
|
@ -42,15 +42,21 @@ def request_fingerprint(request, include_headers=None):
|
|||
the fingeprint. If you want to include specific headers use the
|
||||
include_headers argument, which is a list of Request headers to include.
|
||||
|
||||
Also, servers usually ignore fragments in urls when handling requests,
|
||||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
|
||||
"""
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower())
|
||||
for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
if include_headers not in cache:
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if include_headers:
|
||||
for hdr in include_headers:
|
||||
|
|
@ -58,8 +64,8 @@ def request_fingerprint(request, include_headers=None):
|
|||
fp.update(hdr)
|
||||
for v in request.headers.getlist(hdr):
|
||||
fp.update(v)
|
||||
cache[include_headers] = fp.hexdigest()
|
||||
return cache[include_headers]
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def request_authenticate(request, username, password):
|
||||
|
|
@ -91,4 +97,4 @@ def referer_str(request):
|
|||
referrer = request.headers.get('Referer')
|
||||
if referrer is None:
|
||||
return referrer
|
||||
return to_native_str(referrer, errors='replace')
|
||||
return to_unicode(referrer, errors='replace')
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import webbrowser
|
|||
import tempfile
|
||||
|
||||
from twisted.web import http
|
||||
from scrapy.utils.python import to_bytes, to_native_str
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from w3lib import html
|
||||
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ def response_status_message(status):
|
|||
"""Return status code plus status text descriptive message
|
||||
"""
|
||||
message = http.RESPONSES.get(int(status), "Unknown Status")
|
||||
return '%s %s' % (status, to_native_str(message))
|
||||
return '%s %s' % (status, to_unicode(message))
|
||||
|
||||
|
||||
def response_httprepr(response):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
import OpenSSL
|
||||
import OpenSSL._util as pyOpenSSLutil
|
||||
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
|
||||
|
|
@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
|
|||
|
||||
|
||||
def ffi_buf_to_string(buf):
|
||||
return to_native_str(pyOpenSSLutil.ffi.string(buf))
|
||||
return to_unicode(pyOpenSSLutil.ffi.string(buf))
|
||||
|
||||
|
||||
def x509name_to_string(x509name):
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ singletons members of that object, as explained below:
|
|||
``scrapy.core.manager.ExecutionManager``) - instantiated with a ``Settings``
|
||||
object
|
||||
|
||||
- **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the constructor)
|
||||
- **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the ``__init__`` method)
|
||||
- **crawler.extensions**: ``scrapy.extension.ExtensionManager`` instance
|
||||
- **crawler.engine**: ``scrapy.core.engine.ExecutionEngine`` instance
|
||||
- ``crawler.engine.scheduler``
|
||||
|
|
@ -55,7 +55,7 @@ singletons members of that object, as explained below:
|
|||
``STATS_CLASS`` setting)
|
||||
- **crawler.log**: Logger class with methods replacing the current
|
||||
``scrapy.log`` functions. Logging would be started (if enabled) on
|
||||
``Crawler`` constructor, so no log starting functions are required.
|
||||
``Crawler`` instantiation, so no log starting functions are required.
|
||||
|
||||
- ``crawler.log.msg``
|
||||
- **crawler.signals**: signal handling
|
||||
|
|
@ -69,12 +69,12 @@ Required code changes after singletons removal
|
|||
==============================================
|
||||
|
||||
All components (extensions, middlewares, etc) will receive this ``Crawler``
|
||||
object in their constructors, and this will be the only mechanism for accessing
|
||||
object in their ``__init__`` methods, and this will be the only mechanism for accessing
|
||||
any other components (as opposed to importing each singleton from their
|
||||
respective module). This will also serve to stabilize the core API, something
|
||||
which we haven't documented so far (partly because of this).
|
||||
|
||||
So, for a typical middleware constructor code, instead of this:
|
||||
So, for a typical middleware ``__init__`` method code, instead of this:
|
||||
|
||||
::
|
||||
|
||||
|
|
@ -125,13 +125,13 @@ Open issues to resolve
|
|||
|
||||
- Should we pass ``Settings`` object to ``ScrapyCommand.add_options()``?
|
||||
- How should spiders access settings?
|
||||
- Option 1. Pass ``Crawler`` object to spider constructors too
|
||||
- Option 1. Pass ``Crawler`` object to spider ``__init__`` methods too
|
||||
- pro: one way to access all components (settings and signals being the
|
||||
most relevant to spiders)
|
||||
- con?: spider code can access (and control) any crawler component -
|
||||
since we don't want to support spiders messing with the crawler (write
|
||||
an extension or spider middleware if you need that)
|
||||
- Option 2. Pass ``Settings`` object to spider constructors, which would
|
||||
- Option 2. Pass ``Settings`` object to spider ``__init__`` methods, which would
|
||||
then be accessed through ``self.settings``, like logging which is accessed
|
||||
through ``self.log``
|
||||
|
||||
|
|
|
|||
8
setup.py
8
setup.py
|
|
@ -50,22 +50,20 @@ setup(
|
|||
'License :: OSI Approved :: BSD License',
|
||||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 2',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Programming Language :: Python :: Implementation :: PyPy',
|
||||
'Topic :: Internet :: WWW/HTTP',
|
||||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
|
||||
python_requires='>=3.5',
|
||||
install_requires=[
|
||||
'Twisted>=16.0.0;python_version=="2.7"',
|
||||
'Twisted>=17.9.0;python_version>="3.5"',
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
'cssselect>=0.9.1',
|
||||
'lxml>=3.5.0',
|
||||
|
|
|
|||
|
|
@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
|
|||
os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
|
||||
os.environ['COV_CORE_CONFIG'])
|
||||
|
||||
try:
|
||||
import unittest.mock as mock
|
||||
except ImportError:
|
||||
import mock
|
||||
|
||||
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
|
||||
'sample_data')
|
||||
|
||||
|
|
@ -35,18 +30,3 @@ def get_testdata(*paths):
|
|||
path = os.path.join(tests_datadir, *paths)
|
||||
with open(path, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
# FIXME: delete after dropping py2 support
|
||||
# Monkey patch the unittest module to prevent the
|
||||
# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
|
||||
import six
|
||||
if six.PY2:
|
||||
import unittest
|
||||
import twisted.trial.unittest
|
||||
if not getattr(unittest.TestCase, 'assertRegex', None):
|
||||
unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
|
||||
if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
|
||||
unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
|
||||
if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
|
||||
twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp
|
||||
|
|
|
|||
|
|
@ -206,8 +206,7 @@ class MockServer():
|
|||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.proc.kill()
|
||||
self.proc.wait()
|
||||
time.sleep(0.2)
|
||||
self.proc.communicate()
|
||||
|
||||
def url(self, path, is_secure=False):
|
||||
host = self.http_address.replace('0.0.0.0', '127.0.0.1')
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
# Tests requirements
|
||||
brotlipy
|
||||
jmespath
|
||||
mitmproxy==0.10.1
|
||||
mock
|
||||
netlib==0.10.1
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-twisted
|
||||
pytest-xdist
|
||||
testfixtures
|
||||
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
ipython<6.0
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
# Tests requirements
|
||||
jmespath
|
||||
leveldb; sys_platform != "win32"
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-twisted
|
||||
pytest-xdist
|
||||
sybil
|
||||
testfixtures
|
||||
|
||||
# optional for shell wrapper tests
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue