mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into pylint
This commit is contained in:
Adrián Chaves
GitHub
commit
8c5a8a02ae
|
|
@ -0,0 +1,18 @@
|
|||
skips:
|
||||
- B101
|
||||
- B105
|
||||
- B301
|
||||
- B303
|
||||
- B306
|
||||
- B307
|
||||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
- B406
|
||||
- B410
|
||||
- B503
|
||||
- B603
|
||||
- B605
|
||||
|
|
@ -7,15 +7,16 @@ branches:
|
|||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=security
|
||||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pylint
|
||||
python: 3.8
|
||||
- env: TOXENV=pypy3
|
||||
python: 3.5
|
||||
- env: TOXENV=py35
|
||||
python: 3.5
|
||||
- env: TOXENV=py35-pinned
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5
|
||||
- env: TOXENV=py36
|
||||
python: 3.6
|
||||
|
|
@ -23,7 +24,7 @@ matrix:
|
|||
python: 3.7
|
||||
- env: TOXENV=py38
|
||||
python: 3.8
|
||||
- env: TOXENV=py38-extra-deps
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.6
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ members of the project's leadership.
|
|||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at [http://contributor-covenant.org/version/1/4][version]
|
||||
available at [http://contributor-covenant.org/version/1/4][version].
|
||||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
|
|
|
|||
18
README.rst
18
README.rst
|
|
@ -34,8 +34,8 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
|
|||
crawl websites and extract structured data from their pages. It can be used for
|
||||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
For more information including a list of features check the Scrapy homepage at:
|
||||
https://scrapy.org
|
||||
Check the Scrapy homepage at https://scrapy.org for more information,
|
||||
including a list of features.
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
|
@ -50,8 +50,8 @@ The quick way::
|
|||
|
||||
pip install scrapy
|
||||
|
||||
For more details see the install section in the documentation:
|
||||
https://docs.scrapy.org/en/latest/intro/install.html
|
||||
See the install section in the documentation at
|
||||
https://docs.scrapy.org/en/latest/intro/install.html for more details.
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
|
@ -62,17 +62,17 @@ directory.
|
|||
Releases
|
||||
========
|
||||
|
||||
You can find release notes at https://docs.scrapy.org/en/latest/news.html
|
||||
You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
|
||||
|
||||
Community (blog, twitter, mail list, IRC)
|
||||
=========================================
|
||||
|
||||
See https://scrapy.org/community/
|
||||
See https://scrapy.org/community/ for details.
|
||||
|
||||
Contributing
|
||||
============
|
||||
|
||||
See https://docs.scrapy.org/en/master/contributing.html
|
||||
See https://docs.scrapy.org/en/master/contributing.html for details.
|
||||
|
||||
Code of Conduct
|
||||
---------------
|
||||
|
|
@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
|
|||
Companies using Scrapy
|
||||
======================
|
||||
|
||||
See https://scrapy.org/companies/
|
||||
See https://scrapy.org/companies/ for a list.
|
||||
|
||||
Commercial Support
|
||||
==================
|
||||
|
||||
See https://scrapy.org/support/
|
||||
See https://scrapy.org/support/ for details.
|
||||
|
|
|
|||
19
conftest.py
19
conftest.py
|
|
@ -1,18 +1,23 @@
|
|||
import six
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _py_files(folder):
|
||||
return (str(p) for p in Path(folder).rglob('*.py'))
|
||||
|
||||
|
||||
collect_ignore = [
|
||||
# not a test, but looks like a test
|
||||
"scrapy/utils/testsite.py",
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
||||
*_py_files("tests/CrawlerProcess")
|
||||
]
|
||||
|
||||
|
||||
if six.PY3:
|
||||
for line in open('tests/py3-ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
for line in open('tests/ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,281 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Quotes to Scrape</title>
|
||||
<link rel="stylesheet" href="/static/bootstrap.min.css">
|
||||
<link rel="stylesheet" href="/static/main.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="row header-box">
|
||||
<div class="col-md-8">
|
||||
<h1>
|
||||
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
|
||||
</h1>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<p>
|
||||
|
||||
<a href="/login">Login</a>
|
||||
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-8">
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
|
||||
|
||||
<a class="tag" href="/tag/change/page/1/">change</a>
|
||||
|
||||
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
|
||||
|
||||
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
|
||||
|
||||
<a class="tag" href="/tag/world/page/1/">world</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
|
||||
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
|
||||
<a href="/author/J-K-Rowling">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
|
||||
|
||||
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
|
||||
|
||||
<a class="tag" href="/tag/choices/page/1/">choices</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||
|
||||
<a class="tag" href="/tag/live/page/1/">live</a>
|
||||
|
||||
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
|
||||
|
||||
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
|
||||
<span>by <small class="author" itemprop="author">Jane Austen</small>
|
||||
<a href="/author/Jane-Austen">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
|
||||
|
||||
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
|
||||
|
||||
<a class="tag" href="/tag/books/page/1/">books</a>
|
||||
|
||||
<a class="tag" href="/tag/classic/page/1/">classic</a>
|
||||
|
||||
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</span>
|
||||
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
|
||||
<a href="/author/Marilyn-Monroe">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
|
||||
|
||||
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
|
||||
<span>by <small class="author" itemprop="author">Albert Einstein</small>
|
||||
<a href="/author/Albert-Einstein">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
|
||||
|
||||
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
|
||||
|
||||
<a class="tag" href="/tag/success/page/1/">success</a>
|
||||
|
||||
<a class="tag" href="/tag/value/page/1/">value</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
|
||||
<span>by <small class="author" itemprop="author">André Gide</small>
|
||||
<a href="/author/Andre-Gide">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="life,love" / >
|
||||
|
||||
<a class="tag" href="/tag/life/page/1/">life</a>
|
||||
|
||||
<a class="tag" href="/tag/love/page/1/">love</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“I have not failed. I've just found 10,000 ways that won't work.”</span>
|
||||
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
|
||||
<a href="/author/Thomas-A-Edison">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
|
||||
|
||||
<a class="tag" href="/tag/edison/page/1/">edison</a>
|
||||
|
||||
<a class="tag" href="/tag/failure/page/1/">failure</a>
|
||||
|
||||
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
|
||||
|
||||
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</span>
|
||||
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
|
||||
<a href="/author/Eleanor-Roosevelt">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
|
||||
|
||||
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
|
||||
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
|
||||
<span>by <small class="author" itemprop="author">Steve Martin</small>
|
||||
<a href="/author/Steve-Martin">(about)</a>
|
||||
</span>
|
||||
<div class="tags">
|
||||
Tags:
|
||||
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
|
||||
|
||||
<a class="tag" href="/tag/humor/page/1/">humor</a>
|
||||
|
||||
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
|
||||
|
||||
<a class="tag" href="/tag/simile/page/1/">simile</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav>
|
||||
<ul class="pager">
|
||||
|
||||
|
||||
<li class="next">
|
||||
<a href="/page/2/">Next <span aria-hidden="true">→</span></a>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="col-md-4 tags-box">
|
||||
|
||||
<h2>Top Ten tags</h2>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
|
||||
</span>
|
||||
|
||||
<span class="tag-item">
|
||||
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
|
||||
</span>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<footer class="footer">
|
||||
<div class="container">
|
||||
<p class="text-muted">
|
||||
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
|
||||
</p>
|
||||
<p class="copyright">
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
|
||||
</p>
|
||||
</div>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
25
docs/conf.py
25
docs/conf.py
|
|
@ -12,6 +12,7 @@
|
|||
# serve to show the default.
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from os import path
|
||||
|
||||
# If your extensions are in another directory, add it here. If the directory
|
||||
|
|
@ -27,10 +28,13 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
|
|||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = [
|
||||
'hoverxref.extension',
|
||||
'notfound.extension',
|
||||
'scrapydocs',
|
||||
'sphinx.ext.autodoc',
|
||||
'sphinx.ext.coverage',
|
||||
'sphinx.ext.intersphinx',
|
||||
'sphinx.ext.viewcode',
|
||||
]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
|
|
@ -46,8 +50,8 @@ source_suffix = '.rst'
|
|||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'Scrapy'
|
||||
copyright = u'2008–2018, Scrapy developers'
|
||||
project = 'Scrapy'
|
||||
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
|
|
@ -191,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
|
|||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, document class [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'Scrapy.tex', u'Scrapy Documentation',
|
||||
u'Scrapy developers', 'manual'),
|
||||
('index', 'Scrapy.tex', 'Scrapy Documentation',
|
||||
'Scrapy developers', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
|
|
@ -237,7 +241,7 @@ coverage_ignore_pyobjects = [
|
|||
r'\bContractsManager\b$',
|
||||
|
||||
# For default contracts we only want to document their general purpose in
|
||||
# their constructor, the methods they reimplement to achieve that purpose
|
||||
# their __init__ method, the methods they reimplement to achieve that purpose
|
||||
# should be irrelevant to developers using those contracts.
|
||||
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
|
||||
|
||||
|
|
@ -272,5 +276,16 @@ coverage_ignore_pyobjects = [
|
|||
# -------------------------------------
|
||||
|
||||
intersphinx_mapping = {
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
|
||||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
}
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
# ------------------------------------
|
||||
|
||||
hoverxref_auto_ref = True
|
||||
|
|
|
|||
|
|
@ -1,16 +1,29 @@
|
|||
from doctest import ELLIPSIS
|
||||
import os
|
||||
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
||||
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from sybil import Sybil
|
||||
from sybil.parsers.codeblock import CodeBlockParser
|
||||
from sybil.parsers.doctest import DocTestParser
|
||||
from sybil.parsers.skip import skip
|
||||
|
||||
|
||||
def load_response(url, filename):
|
||||
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
|
||||
with open(input_path, 'rb') as input_file:
|
||||
return HtmlResponse(url, body=input_file.read())
|
||||
|
||||
|
||||
def setup(namespace):
|
||||
namespace['load_response'] = load_response
|
||||
|
||||
|
||||
pytest_collect_file = Sybil(
|
||||
parsers=[
|
||||
DocTestParser(optionflags=ELLIPSIS),
|
||||
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||
CodeBlockParser(future_imports=['print_function']),
|
||||
skip,
|
||||
],
|
||||
pattern='*.rst',
|
||||
setup=setup,
|
||||
).pytest()
|
||||
|
|
|
|||
|
|
@ -177,79 +177,71 @@ Documentation policies
|
|||
======================
|
||||
|
||||
For reference documentation of API members (classes, methods, etc.) use
|
||||
docstrings and make sure that the Sphinx documentation uses the autodoc_
|
||||
extension to pull the docstrings. API reference documentation should follow
|
||||
docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
|
||||
and it may provide short examples.
|
||||
docstrings and make sure that the Sphinx documentation uses the
|
||||
:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
|
||||
documentation should follow docstring conventions (`PEP 257`_) and be
|
||||
IDE-friendly: short, to the point, and it may provide short examples.
|
||||
|
||||
Other types of documentation, such as tutorials or topics, should be covered in
|
||||
files within the ``docs/`` directory. This includes documentation that is
|
||||
specific to an API member, but goes beyond API reference documentation.
|
||||
|
||||
In any case, if something is covered in a docstring, use the autodoc_
|
||||
extension to pull the docstring into the documentation instead of duplicating
|
||||
the docstring in files within the ``docs/`` directory.
|
||||
|
||||
.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
|
||||
In any case, if something is covered in a docstring, use the
|
||||
:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
|
||||
documentation instead of duplicating the docstring in files within the
|
||||
``docs/`` directory.
|
||||
|
||||
Tests
|
||||
=====
|
||||
|
||||
Tests are implemented using the `Twisted unit-testing framework`_, running
|
||||
tests requires `tox`_.
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:core/development/policy/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
|
||||
.. _running-tests:
|
||||
|
||||
Running tests
|
||||
-------------
|
||||
|
||||
Make sure you have a recent enough `tox`_ installation:
|
||||
To run all tests::
|
||||
|
||||
``tox --version``
|
||||
|
||||
If your version is older than 1.7.0, please update it first:
|
||||
|
||||
``pip install -U tox``
|
||||
|
||||
To run all tests go to the root directory of Scrapy source code and run:
|
||||
|
||||
``tox``
|
||||
tox
|
||||
|
||||
To run a specific test (say ``tests/test_loader.py``) use:
|
||||
|
||||
``tox -- tests/test_loader.py``
|
||||
|
||||
To run the tests on a specific tox_ environment, use ``-e <name>`` with an
|
||||
environment name from ``tox.ini``. For example, to run the tests with Python
|
||||
3.6 use::
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
|
||||
tox -e py36
|
||||
|
||||
You can also specify a comma-separated list of environmets, and use `tox’s
|
||||
parallel mode`_ to run the tests on multiple environments in parallel::
|
||||
You can also specify a comma-separated list of environmets, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py27,py36 -p auto
|
||||
tox -e py36,py38 -p auto
|
||||
|
||||
To pass command-line options to pytest_, add them after ``--`` in your call to
|
||||
tox_. Using ``--`` overrides the default positional arguments defined in
|
||||
``tox.ini``, so you must include those default positional arguments
|
||||
(``scrapy tests``) after ``--`` as well::
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
default positional arguments defined in ``tox.ini``, so you must include those
|
||||
default positional arguments (``scrapy tests``) after ``--`` as well::
|
||||
|
||||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 tox_ environment using all your CPU cores::
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install `coverage`_ (``pip install coverage``) and run:
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
||||
``coverage report``
|
||||
|
||||
see output of ``coverage --help`` for more options like html or xml report.
|
||||
|
||||
.. _coverage: https://pypi.python.org/pypi/coverage
|
||||
|
||||
Writing tests
|
||||
-------------
|
||||
|
||||
|
|
@ -270,13 +262,9 @@ And their unit-tests are in::
|
|||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://reddit.com/r/scrapy
|
||||
.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
|
||||
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
|
||||
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
|
||||
.. _open issues: https://github.com/scrapy/scrapy/issues
|
||||
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
|
||||
.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
|
||||
.. _pytest: https://docs.pytest.org/en/latest/usage.html
|
||||
.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
|
||||
.. _tox: https://pypi.python.org/pypi/tox
|
||||
.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
|
||||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.xpath('span/small/text()').get(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr("href")').get()
|
||||
|
|
|
|||
|
|
@ -235,13 +235,16 @@ You will see something like::
|
|||
[s] shelp() Shell help (print this help)
|
||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||
[s] view(response) View response in a browser
|
||||
>>>
|
||||
|
||||
Using the shell, you can try selecting elements using `CSS`_ with the response
|
||||
object::
|
||||
object:
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
||||
The result of running ``response.css('title')`` is a list-like object called
|
||||
:class:`~scrapy.selector.SelectorList`, which represents a list of
|
||||
|
|
@ -372,6 +375,9 @@ we want::
|
|||
We get a list of selectors for the quote HTML elements with::
|
||||
|
||||
>>> response.css("div.quote")
|
||||
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
|
||||
...]
|
||||
|
||||
Each of the selectors returned by the query above allows us to run further
|
||||
queries over their sub-elements. Let's assign the first selector to a
|
||||
|
|
@ -396,6 +402,12 @@ to get all of them::
|
|||
>>> tags
|
||||
['change', 'deep-thoughts', 'thinking', 'world']
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from sys import version_info
|
||||
|
||||
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
|
||||
|
||||
Having figured out how to extract each bit, we can now iterate over all the
|
||||
quotes elements and put them together into a Python dictionary::
|
||||
|
||||
|
|
@ -404,10 +416,9 @@ quotes elements and put them together into a Python dictionary::
|
|||
... author = quote.css("small.author::text").get()
|
||||
... tags = quote.css("div.tags a.tag::text").getall()
|
||||
... print(dict(text=text, author=author, tags=tags))
|
||||
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
|
||||
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
|
||||
... a few more of these, omitted for brevity
|
||||
>>>
|
||||
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
|
||||
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
|
||||
...
|
||||
|
||||
Extracting data in our spider
|
||||
-----------------------------
|
||||
|
|
@ -521,7 +532,7 @@ There is also an ``attrib`` property available
|
|||
(see :ref:`selecting-attributes` for more)::
|
||||
|
||||
>>> response.css('li.next a').attrib['href']
|
||||
'/page/2'
|
||||
'/page/2/'
|
||||
|
||||
Let's see now our spider modified to recursively follow the link to the next
|
||||
page, extracting data from it::
|
||||
|
|
|
|||
|
|
@ -308,12 +308,12 @@ New features
|
|||
convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
|
||||
|
||||
* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
|
||||
constructor now receives the :class:`~scrapy.http.Response` object that
|
||||
``__init__`` method now receives the :class:`~scrapy.http.Response` object that
|
||||
originated the request as its second argument (:issue:`3682`)
|
||||
|
||||
* A new ``restrict_text`` parameter for the
|
||||
:attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
constructor allows filtering links by linking text (:issue:`3622`,
|
||||
``__init__`` method allows filtering links by linking text (:issue:`3622`,
|
||||
:issue:`3635`)
|
||||
|
||||
* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
|
||||
|
|
@ -479,7 +479,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
|
|||
|
||||
* From :class:`~scrapy.selector.Selector`:
|
||||
|
||||
* ``_root`` (both the constructor argument and the object property, use
|
||||
* ``_root`` (both the ``__init__`` method argument and the object property, use
|
||||
``root``)
|
||||
|
||||
* ``extract_unquoted`` (use ``getall``)
|
||||
|
|
@ -2703,7 +2703,7 @@ Scrapy changes:
|
|||
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
|
||||
- promoted :ref:`topics-djangoitem` to main contrib
|
||||
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
|
||||
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
|
||||
- removed signal: ``scrapy.mail.mail_sent``
|
||||
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
|
||||
|
|
@ -2917,7 +2917,7 @@ API changes
|
|||
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
|
||||
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
|
||||
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
|
||||
- Changes to Scrapy Manager (now called "Crawler"):
|
||||
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
|
||||
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
|
||||
|
|
|
|||
|
|
@ -1,2 +1,4 @@
|
|||
Sphinx>=2.1
|
||||
sphinx_rtd_theme
|
||||
sphinx-hoverxref
|
||||
sphinx-notfound-page
|
||||
sphinx_rtd_theme
|
||||
|
|
|
|||
|
|
@ -273,5 +273,3 @@ class (which they all inherit from).
|
|||
|
||||
Close the given spider. After this is called, no more specific stats
|
||||
can be accessed or collected.
|
||||
|
||||
.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
|
|
|
|||
|
|
@ -166,11 +166,10 @@ for concurrency.
|
|||
For more information about asynchronous programming and Twisted see these
|
||||
links:
|
||||
|
||||
* `Introduction to Deferreds in Twisted`_
|
||||
* :doc:`twisted:core/howto/defer-intro`
|
||||
* `Twisted - hello, asynchronous programming`_
|
||||
* `Twisted Introduction - Krondo`_
|
||||
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. highlight:: none
|
||||
|
||||
.. _topics-commands:
|
||||
|
||||
=================
|
||||
|
|
@ -66,7 +68,9 @@ structure by default, similar to this::
|
|||
|
||||
The directory where the ``scrapy.cfg`` file resides is known as the *project
|
||||
root directory*. That file contains the name of the python module that defines
|
||||
the project settings. Here is an example::
|
||||
the project settings. Here is an example:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[settings]
|
||||
default = myproject.settings
|
||||
|
|
@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
|
|||
shared by multiple Scrapy projects, each with its own settings module.
|
||||
|
||||
In that case, you must define one or more aliases for those settings modules
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file::
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[settings]
|
||||
default = myproject1.settings
|
||||
|
|
@ -277,6 +283,8 @@ check
|
|||
|
||||
Run contract checks.
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage examples::
|
||||
|
||||
$ scrapy check -l
|
||||
|
|
@ -294,6 +302,8 @@ Usage examples::
|
|||
[FAILED] first_spider:parse
|
||||
>>> Returned 92 requests, expected 0..4
|
||||
|
||||
.. skip: end
|
||||
|
||||
.. command:: list
|
||||
|
||||
list
|
||||
|
|
@ -481,6 +491,8 @@ Supported options:
|
|||
|
||||
* ``--verbose`` or ``-v``: display information for each depth level
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage example::
|
||||
|
||||
$ scrapy parse http://www.example.com/ -c parse_item
|
||||
|
|
@ -495,6 +507,8 @@ Usage example::
|
|||
# Requests -----------------------------------------------------------------
|
||||
[]
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
||||
.. command:: settings
|
||||
|
||||
|
|
@ -573,7 +587,9 @@ Default: ``''`` (empty string)
|
|||
A module to use for looking up custom Scrapy commands. This is used to add custom
|
||||
commands for your Scrapy project.
|
||||
|
||||
Example::
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
COMMANDS_MODULE = 'mybot.commands'
|
||||
|
||||
|
|
@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
|
|||
``scrapy.commands`` section in the entry points of the library ``setup.py``
|
||||
file.
|
||||
|
||||
The following example adds ``my_command`` command::
|
||||
The following example adds ``my_command`` command:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
|
|||
of the spider at the method level. It has the advantage of being flexible and
|
||||
simple to use, but does not allow debugging code inside a method.
|
||||
|
||||
.. highlight:: none
|
||||
|
||||
.. skip: start
|
||||
|
||||
In order to see the item scraped from a specific url::
|
||||
|
||||
$ scrapy parse --spider=myspider -c parse_item -d 2 <item_url>
|
||||
|
|
@ -85,6 +89,8 @@ using::
|
|||
|
||||
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
||||
Scrapy Shell
|
||||
============
|
||||
|
|
@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides
|
|||
showing the response received and the output. How to debug the situation when
|
||||
``parse_details`` sometimes receives no item?
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
Fortunately, the :command:`shell` is your bread and butter in this case (see
|
||||
:ref:`topics-shell-inspect-response`)::
|
||||
|
||||
|
|
|
|||
|
|
@ -474,7 +474,7 @@ DBM storage backend
|
|||
|
||||
A DBM_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
By default, it uses the anydbm_ module, but you can change it with the
|
||||
By default, it uses the :mod:`dbm`, but you can change it with the
|
||||
:setting:`HTTPCACHE_DBM_MODULE` setting.
|
||||
|
||||
.. _httpcache-storage-custom:
|
||||
|
|
@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
|
|||
|
||||
.. versionadded:: 0.13
|
||||
|
||||
Default: ``'anydbm'``
|
||||
Default: ``'dbm'``
|
||||
|
||||
The database module to use in the :ref:`DBM storage backend
|
||||
<httpcache-storage-dbm>`. This setting is specific to the DBM backend.
|
||||
|
|
@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
|
|||
|
||||
|
||||
.. _DBM: https://en.wikipedia.org/wiki/Dbm
|
||||
.. _anydbm: https://docs.python.org/2/library/anydbm.html
|
||||
|
|
|
|||
|
|
@ -9,19 +9,19 @@ Sending e-mail
|
|||
|
||||
Although Python makes sending e-mails relatively easy via the `smtplib`_
|
||||
library, Scrapy provides its own facility for sending e-mails which is very
|
||||
easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
|
||||
interfering with the non-blocking IO of the crawler. It also provides a
|
||||
simple API for sending attachments and it's very easy to configure, with a few
|
||||
:ref:`settings <topics-email-settings>`.
|
||||
easy to use and it's implemented using :doc:`Twisted non-blocking IO
|
||||
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
|
||||
IO of the crawler. It also provides a simple API for sending attachments and
|
||||
it's very easy to configure, with a few :ref:`settings
|
||||
<topics-email-settings>`.
|
||||
|
||||
.. _smtplib: https://docs.python.org/2/library/smtplib.html
|
||||
.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
|
||||
|
||||
Quick example
|
||||
=============
|
||||
|
||||
There are two ways to instantiate the mail sender. You can instantiate it using
|
||||
the standard constructor::
|
||||
the standard ``__init__`` method::
|
||||
|
||||
from scrapy.mail import MailSender
|
||||
mailer = MailSender()
|
||||
|
|
@ -39,7 +39,8 @@ MailSender class reference
|
|||
==========================
|
||||
|
||||
MailSender is the preferred class to use for sending emails from Scrapy, as it
|
||||
uses `Twisted non-blocking IO`_, like the rest of the framework.
|
||||
uses :doc:`Twisted non-blocking IO <twisted:core/howto/defer-intro>`, like the
|
||||
rest of the framework.
|
||||
|
||||
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
|
||||
|
||||
|
|
@ -111,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
|
|||
Mail settings
|
||||
=============
|
||||
|
||||
These settings define the default constructor values of the :class:`MailSender`
|
||||
These settings define the default ``__init__`` method values of the :class:`MailSender`
|
||||
class, and can be used to configure e-mail notifications in your project without
|
||||
writing any code (for those extensions and code that uses :class:`MailSender`).
|
||||
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ described next.
|
|||
1. Declaring a serializer in the field
|
||||
--------------------------------------
|
||||
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
a callable which receives a value and returns its serialized form.
|
||||
|
||||
Example::
|
||||
|
|
@ -144,7 +144,7 @@ BaseItemExporter
|
|||
defining what fields to export, whether to export empty fields, or which
|
||||
encoding to use.
|
||||
|
||||
These features can be configured through the constructor arguments which
|
||||
These features can be configured through the ``__init__`` method arguments which
|
||||
populate their respective instance attributes: :attr:`fields_to_export`,
|
||||
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
|
||||
|
||||
|
|
@ -246,8 +246,8 @@ XmlItemExporter
|
|||
:param item_element: The name of each item element in the exported XML.
|
||||
:type item_element: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -306,9 +306,9 @@ CsvItemExporter
|
|||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor, and the leftover arguments to the
|
||||
`csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
|
||||
argument to customize this exporter.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
|
@ -334,8 +334,8 @@ PickleItemExporter
|
|||
|
||||
For more information, refer to the `pickle module documentation`_.
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
Pickle isn't a human readable format, so no output examples are provided.
|
||||
|
||||
|
|
@ -351,8 +351,8 @@ PprintItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -367,10 +367,10 @@ JsonItemExporter
|
|||
.. class:: JsonItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional constructor arguments are
|
||||
passed to the :class:`BaseItemExporter` constructor, and the leftover
|
||||
arguments to the `JSONEncoder`_ constructor, so you can use any
|
||||
`JSONEncoder`_ constructor argument to customize this exporter.
|
||||
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
|
||||
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -398,10 +398,10 @@ JsonLinesItemExporter
|
|||
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional constructor arguments are passed
|
||||
to the :class:`BaseItemExporter` constructor, and the leftover arguments to
|
||||
the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
|
||||
constructor argument to customize this exporter.
|
||||
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
|
||||
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
|
||||
``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ Loading & activating extensions
|
|||
|
||||
Extensions are loaded and activated at startup by instantiating a single
|
||||
instance of the extension class. Therefore, all the extension initialization
|
||||
code must be performed in the class constructor (``__init__`` method).
|
||||
code must be performed in the class ``__init__`` method.
|
||||
|
||||
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
|
||||
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
|
|||
|
||||
This method is called for every item pipeline component. :meth:`process_item`
|
||||
must either: return a dict with data, return an :class:`~scrapy.item.Item`
|
||||
(or any descendant class) object, return a `Twisted Deferred`_ or raise
|
||||
(or any descendant class) object, return a
|
||||
:class:`~twisted.internet.defer.Deferred` or raise
|
||||
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
|
||||
processed by further pipeline components.
|
||||
|
||||
|
|
@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
|
|||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
||||
.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
Item pipeline example
|
||||
=====================
|
||||
|
||||
|
|
@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
|
|||
Take screenshot of item
|
||||
-----------------------
|
||||
|
||||
This example demonstrates how to return Deferred_ from :meth:`process_item` method.
|
||||
This example demonstrates how to return a
|
||||
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
|
||||
It uses Splash_ to render screenshot of item url. Pipeline
|
||||
makes request to locally running instance of Splash_. After request is downloaded
|
||||
and Deferred callback fires, it saves item to a file and adds filename to an item.
|
||||
|
|
@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
return item
|
||||
|
||||
.. _Splash: https://splash.readthedocs.io/en/stable/
|
||||
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
Duplicates filter
|
||||
-----------------
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ especially in a larger project with many spiders.
|
|||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide a `dictionary-like`_ API with a convenient syntax for declaring
|
||||
their available fields.
|
||||
their available fields.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
serialization can be customized using Item fields metadata, :mod:`trackref`
|
||||
tracks Item instances to help find memory leaks
|
||||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
|
||||
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
|
@ -237,7 +237,7 @@ Item objects
|
|||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
|
||||
Items replicate the standard `dict API`_, including its constructor, and
|
||||
Items replicate the standard `dict API`_, including its ``__init__`` method, and
|
||||
also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Using Item Loaders to populate items
|
|||
|
||||
To use an Item Loader, you must first instantiate it. You can either
|
||||
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
|
||||
which case an Item is automatically instantiated in the Item Loader constructor
|
||||
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
|
||||
using the Item class specified in the :attr:`ItemLoader.default_item_class`
|
||||
attribute.
|
||||
|
||||
|
|
@ -142,20 +142,6 @@ accept one (and only one) positional argument, which will be an iterable.
|
|||
containing the collected values (for that field). The result of the output
|
||||
processors is the value that will be finally assigned to the item.
|
||||
|
||||
If you want to use a plain function as a processor, make sure it receives
|
||||
``self`` as the first argument::
|
||||
|
||||
def lowercase_processor(self, values):
|
||||
for v in values:
|
||||
yield v.lower()
|
||||
|
||||
class MyItemLoader(ItemLoader):
|
||||
name_in = lowercase_processor
|
||||
|
||||
This is because whenever a function is assigned as a class variable, it becomes
|
||||
a method and would be passed the instance as the the first argument when being
|
||||
called. See `this answer on stackoverflow`_ for more details.
|
||||
|
||||
The other thing you need to keep in mind is that the values returned by input
|
||||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
|
@ -163,7 +149,7 @@ processors to populate the fields.
|
|||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
|
||||
.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
|
||||
|
||||
|
||||
Declaring Item Loaders
|
||||
======================
|
||||
|
|
@ -271,7 +257,7 @@ There are several ways to modify Item Loader context values:
|
|||
loader.context['unit'] = 'cm'
|
||||
|
||||
2. On Item Loader instantiation (the keyword arguments of Item Loader
|
||||
constructor are stored in the Item Loader context)::
|
||||
``__init__`` method are stored in the Item Loader context)::
|
||||
|
||||
loader = ItemLoader(product, unit='cm')
|
||||
|
||||
|
|
@ -491,6 +477,8 @@ ItemLoader objects
|
|||
.. attribute:: item
|
||||
|
||||
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
|
||||
This is mostly used as a property so when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
|
|
@ -500,7 +488,7 @@ ItemLoader objects
|
|||
.. attribute:: default_item_class
|
||||
|
||||
An Item class (or factory), used to instantiate items when not given in
|
||||
the constructor.
|
||||
the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
|
|
@ -515,15 +503,15 @@ ItemLoader objects
|
|||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the constructor.
|
||||
If a selector is given in the constructor this attribute is ignored.
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the constructor or one created from
|
||||
the response given in the constructor using the
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
|
||||
|
|
@ -648,7 +636,7 @@ Here is a list of all built-in processors:
|
|||
.. class:: Identity
|
||||
|
||||
The simplest processor, which doesn't do anything. It returns the original
|
||||
values unchanged. It doesn't receive any constructor arguments, nor does it
|
||||
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
|
||||
accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
|
@ -662,7 +650,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
Returns the first non-null/non-empty value from the values received,
|
||||
so it's typically used as an output processor to single-valued fields.
|
||||
It doesn't receive any constructor arguments, nor does it accept Loader contexts.
|
||||
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -673,7 +661,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
.. class:: Join(separator=u' ')
|
||||
|
||||
Returns the values joined with the separator given in the constructor, which
|
||||
Returns the values joined with the separator given in the ``__init__`` method, which
|
||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||
|
||||
When using the default separator, this processor is equivalent to the
|
||||
|
|
@ -711,7 +699,7 @@ Here is a list of all built-in processors:
|
|||
those which do, this processor will pass the currently active :ref:`Loader
|
||||
context <topics-loaders-context>` through that parameter.
|
||||
|
||||
The keyword arguments passed in the constructor are used as the default
|
||||
The keyword arguments passed in the ``__init__`` method are used as the default
|
||||
Loader context values passed to each function call. However, the final
|
||||
Loader context values passed to functions are overridden with the currently
|
||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
|
|
@ -755,12 +743,12 @@ Here is a list of all built-in processors:
|
|||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
constructor keyword arguments are used as default context values. See
|
||||
``__init__`` method keyword arguments are used as default context values. See
|
||||
:class:`Compose` processor for more info.
|
||||
|
||||
.. class:: SelectJmes(json_path)
|
||||
|
||||
Queries the value using the json path provided to the constructor and returns the output.
|
||||
Queries the value using the json path provided to the ``__init__`` method and returns the output.
|
||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||
This processor takes only one input at a time.
|
||||
|
||||
|
|
|
|||
|
|
@ -255,18 +255,18 @@ scrapy.utils.log module
|
|||
when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
|
||||
In that case, its usage is not required but it's recommended.
|
||||
|
||||
If you plan on configuring the handlers yourself is still recommended you
|
||||
call this function, passing ``install_root_handler=False``. Bear in mind
|
||||
there won't be any log output set by default in that case.
|
||||
Another option when running custom scripts is to manually configure the logging.
|
||||
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
|
||||
|
||||
To get you started on manually configuring logging's output, you can use
|
||||
`logging.basicConfig()`_ to set a basic root handler. This is an example
|
||||
on how to redirect ``INFO`` or higher messages to a file::
|
||||
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
|
||||
so it is recommended to only use `logging.basicConfig()`_ together with
|
||||
:class:`~scrapy.crawler.CrawlerRunner`.
|
||||
|
||||
This is an example on how to redirect ``INFO`` or higher messages to a file::
|
||||
|
||||
import logging
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
configure_logging(install_root_handler=False)
|
||||
logging.basicConfig(
|
||||
filename='log.txt',
|
||||
format='%(levelname)s: %(message)s',
|
||||
|
|
|
|||
|
|
@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
* ``success`` is a boolean which is ``True`` if the image was downloaded
|
||||
successfully or ``False`` if it failed for some reason
|
||||
|
||||
* ``file_info_or_error`` is a dict containing the following keys (if success
|
||||
is ``True``) or a `Twisted Failure`_ if there was a problem.
|
||||
* ``file_info_or_error`` is a dict containing the following keys (if
|
||||
success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
|
||||
there was a problem.
|
||||
|
||||
* ``url`` - the url where the file was downloaded from. This is the url of
|
||||
the request returned from the :meth:`~get_media_requests`
|
||||
|
|
@ -577,5 +578,4 @@ above::
|
|||
item['image_paths'] = image_paths
|
||||
return item
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running.
|
|||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run() # the script will block here until the crawling is finished
|
||||
|
||||
.. seealso:: `Twisted Reactor Overview`_.
|
||||
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
|
||||
|
||||
.. _run-multiple-spiders:
|
||||
|
||||
|
|
@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
.. _Crawlera: https://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ Request objects
|
|||
|
||||
:param errback: a function that will be called if any exception was
|
||||
raised while processing the request. This includes pages that failed
|
||||
with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
|
||||
as first parameter.
|
||||
with 404 HTTP errors and such. It receives a
|
||||
:exc:`~twisted.python.failure.Failure` as first parameter.
|
||||
For more information,
|
||||
see :ref:`topics-request-response-ref-errbacks` below.
|
||||
:type errback: callable
|
||||
|
|
@ -137,7 +137,7 @@ Request objects
|
|||
|
||||
A string containing the URL of this request. Keep in mind that this
|
||||
attribute contains the escaped URL, so it can differ from the URL passed in
|
||||
the constructor.
|
||||
the ``__init__`` method.
|
||||
|
||||
This attribute is read-only. To change the URL of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing
|
|||
The errback of a request is a function that will be called when an exception
|
||||
is raise while processing it.
|
||||
|
||||
It receives a `Twisted Failure`_ instance as first parameter and can be
|
||||
used to track connection establishment timeouts, DNS errors etc.
|
||||
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
|
||||
be used to track connection establishment timeouts, DNS errors etc.
|
||||
|
||||
Here's an example spider logging all errors and catching some specific
|
||||
errors if needed::
|
||||
|
|
@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
.. class:: FormRequest(url, [formdata, ...])
|
||||
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
|
|||
:type dont_click: boolean
|
||||
|
||||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` constructor.
|
||||
:class:`FormRequest` ``__init__`` method.
|
||||
|
||||
.. versionadded:: 0.10.3
|
||||
The ``formname`` parameter.
|
||||
|
|
@ -547,7 +547,7 @@ dealing with JSON requests.
|
|||
|
||||
.. class:: JsonRequest(url, [... data, dumps_kwargs])
|
||||
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -556,7 +556,7 @@ dealing with JSON requests.
|
|||
|
||||
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
|
||||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
|
|
@ -721,7 +721,7 @@ TextResponse objects
|
|||
:class:`Response` class, which is meant to be used only for binary data,
|
||||
such as images, sounds or any media file.
|
||||
|
||||
:class:`TextResponse` objects support a new constructor argument, in
|
||||
:class:`TextResponse` objects support a new ``__init__`` method argument, in
|
||||
addition to the base :class:`Response` objects. The remaining functionality
|
||||
is the same as for the :class:`Response` class and is not documented here.
|
||||
|
||||
|
|
@ -755,7 +755,7 @@ TextResponse objects
|
|||
A string with the encoding of this response. The encoding is resolved by
|
||||
trying the following mechanisms, in order:
|
||||
|
||||
1. the encoding passed in the constructor ``encoding`` argument
|
||||
1. the encoding passed in the ``__init__`` method ``encoding`` argument
|
||||
|
||||
2. the encoding declared in the Content-Type HTTP header. If this
|
||||
encoding is not valid (ie. unknown), it is ignored and the next
|
||||
|
|
@ -816,5 +816,4 @@ XmlResponse objects
|
|||
adds encoding auto-discovering support by looking into the XML declaration
|
||||
line. See :attr:`TextResponse.encoding`.
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
|
||||
|
|
|
|||
|
|
@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
|
|||
Deferred signal handlers
|
||||
========================
|
||||
|
||||
Some signals support returning `Twisted deferreds`_ from their handlers, see
|
||||
the :ref:`topics-signals-ref` below to know which ones.
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, see the :ref:`topics-signals-ref` below to know
|
||||
which ones.
|
||||
|
||||
.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
@ -155,8 +155,8 @@ item_error
|
|||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
spider_closed
|
||||
-------------
|
||||
|
|
@ -236,8 +236,8 @@ spider_error
|
|||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
:param response: the response being processed when the exception was raised
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -333,5 +333,3 @@ response_downloaded
|
|||
|
||||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
from __future__ import print_function
|
||||
from time import time
|
||||
from collections import deque
|
||||
from twisted.web.server import Site, NOT_DONE_YET
|
||||
|
|
|
|||
241
pytest.ini
241
pytest.ini
|
|
@ -7,10 +7,7 @@ addopts =
|
|||
--doctest-modules
|
||||
--ignore=docs/_ext
|
||||
--ignore=docs/conf.py
|
||||
--ignore=docs/intro/tutorial.rst
|
||||
--ignore=docs/news.rst
|
||||
--ignore=docs/topics/commands.rst
|
||||
--ignore=docs/topics/debug.rst
|
||||
--ignore=docs/topics/developer-tools.rst
|
||||
--ignore=docs/topics/dynamic-content.rst
|
||||
--ignore=docs/topics/items.rst
|
||||
|
|
@ -23,250 +20,230 @@ addopts =
|
|||
--ignore=docs/utils
|
||||
twisted = 1
|
||||
flake8-ignore =
|
||||
# Files that are only meant to provide top-level imports are expected not
|
||||
# to use any of their imports:
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
# Issues pending a review:
|
||||
# extras
|
||||
extras/qps-bench-server.py E261 E501
|
||||
extras/qpsclient.py E501 E261 E501
|
||||
extras/qps-bench-server.py E501
|
||||
extras/qpsclient.py E501 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py F401 E501
|
||||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
|
||||
scrapy/commands/fetch.py E401 E501 E128 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/list.py E302
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E302 E128
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/startproject.py E502 E127 E501 E128
|
||||
scrapy/commands/startproject.py E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
scrapy/commands/view.py F401 E302
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E502 E128
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
|
||||
scrapy/core/engine.py E501 E128 E127 E306 E502
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E261 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226
|
||||
scrapy/core/downloader/__init__.py F401 E501
|
||||
scrapy/core/scraper.py E501 E306 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E126 E226
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
|
||||
scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E502 E128
|
||||
scrapy/downloadermiddlewares/httpcompression.py E501 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py F401 E501
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E502 E128 E123
|
||||
scrapy/extensions/corestats.py E302 E501
|
||||
scrapy/extensions/closespider.py E501 E128 E123
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303 F401
|
||||
scrapy/extensions/httpcache.py E128 E501 E303
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E302 E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128 W293 W291
|
||||
scrapy/http/response/html.py E302
|
||||
scrapy/http/response/text.py E501 W293 E128 E124
|
||||
scrapy/http/response/xml.py E302
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
|
||||
scrapy/linkextractors/__init__.py E731 E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E502 E128
|
||||
scrapy/loader/common.py E302
|
||||
scrapy/loader/__init__.py E501 E128
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/__init__.py E302
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
scrapy/pipelines/images.py E265 E501
|
||||
scrapy/pipelines/media.py E125 E501 E266
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403 F401
|
||||
scrapy/selector/unified.py F401 E501 E111
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
|
||||
scrapy/settings/default_settings.py E501 E114 E116 E226
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py F401 E501 E402
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501 E261
|
||||
scrapy/spiders/feed.py E501
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/boto.py F401
|
||||
scrapy/utils/conf.py E402 E502 E501
|
||||
scrapy/utils/console.py E302 E261 F401 E306 E305
|
||||
scrapy/utils/curl.py F401
|
||||
scrapy/utils/conf.py E402 E501
|
||||
scrapy/utils/console.py E306 E305
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/decorators.py E501 E302
|
||||
scrapy/utils/defer.py E501 E302 E128
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/display.py E302
|
||||
scrapy/utils/engine.py F401 E261 E302
|
||||
scrapy/utils/ftp.py E302
|
||||
scrapy/utils/gz.py E305 E501 E302 W504
|
||||
scrapy/utils/http.py F403 F401 E226
|
||||
scrapy/utils/httpobj.py E302 E501
|
||||
scrapy/utils/gz.py E305 E501 W504
|
||||
scrapy/utils/http.py F403 E226
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501 E701
|
||||
scrapy/utils/job.py E302
|
||||
scrapy/utils/log.py E128 W503
|
||||
scrapy/utils/markup.py F403 F401 W292
|
||||
scrapy/utils/markup.py F403 W292
|
||||
scrapy/utils/misc.py E501 E226
|
||||
scrapy/utils/multipart.py F403 F401 W292
|
||||
scrapy/utils/multipart.py F403 W292
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501 E302
|
||||
scrapy/utils/reactor.py E302 E226
|
||||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E226
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E302 E127 E501
|
||||
scrapy/utils/response.py E501 E302 E128
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E271 E302 E501
|
||||
scrapy/utils/spider.py E271 E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/template.py E302
|
||||
scrapy/utils/test.py E302 E501
|
||||
scrapy/utils/url.py E501 F403 F401 E128 F405
|
||||
scrapy/utils/test.py E501
|
||||
scrapy/utils/url.py E501 F403 E128 F405
|
||||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/_monkeypatches.py W293
|
||||
scrapy/cmdline.py E502 E501
|
||||
scrapy/cmdline.py E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E302 E501 E202
|
||||
scrapy/exceptions.py E302 E501
|
||||
scrapy/exporters.py E501 E261 E226
|
||||
scrapy/extension.py E302
|
||||
scrapy/interfaces.py E302 E501
|
||||
scrapy/dupefilters.py E501 E202
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501 E226
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501 W293
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/middleware.py E502 E128 E501
|
||||
scrapy/middleware.py E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E302
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/robotstxt.py E302 E501
|
||||
scrapy/robotstxt.py E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py E225 F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py F401 E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123 F401
|
||||
tests/pipelines.py E302 F841 E226
|
||||
tests/spiders.py E302 E501 E127
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/pipelines.py F841 E226
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501 E261
|
||||
tests/test_command_parse.py F401 E302 E501 E128 E303 E226
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128 E303 E226
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py F401 E128 E501
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128 W293
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py E302 F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E711 E126 E226 E123
|
||||
tests/test_dependencies.py F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E713 E501 E302 E305 F401
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
|
||||
tests/test_downloadermiddleware_httpcache.py E501 E305
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E502 E128 E261
|
||||
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_extension_telnet.py F401 F841
|
||||
tests/test_feedexport.py E501 F401 F841 E241
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E302 E501
|
||||
tests/test_http_request.py F401 E402 E501 E231 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_response.py E501 E301 E502 E128 E265
|
||||
tests/test_item.py E701 E128 E231 F841 E306
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123
|
||||
tests/test_http_response.py E501 E301 E128 E265
|
||||
tests/test_item.py E701 E128 F841 E306
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E231 E124
|
||||
tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E231 E122 E302
|
||||
tests/test_mail.py E302 E128 E501 E305
|
||||
tests/test_middleware.py E302 E501 E128
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501 E305
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F401 F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
|
||||
tests/test_pipeline_files.py E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E302 E305
|
||||
tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
|
||||
tests/test_responsetypes.py E501 E305
|
||||
tests/test_robotstxt_interface.py E501 W291 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py F401 E501 E127
|
||||
tests/test_spider.py E501 F401
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501 E226
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
|
||||
tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E302 E701 E741
|
||||
tests/test_utils_conf.py E501 E231 E303 E128
|
||||
tests/test_utils_console.py E302 E231
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py E501 W293 E226
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E701 E741
|
||||
tests/test_utils_conf.py E501 E303 E128
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
|
||||
tests/test_utils_defer.py E306 E501 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_http.py E302 E501 E502 E128 W504
|
||||
tests/test_utils_httpobj.py E302
|
||||
tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
tests/test_utils_reqser.py F401 E501 E128
|
||||
tests/test_utils_request.py E302 E501 E128 E305
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128 E305
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E302 E731 E226
|
||||
tests/test_utils_sitemap.py E302 E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E302 E305
|
||||
tests/test_utils_signal.py E741 F841 E731 E226
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_spider.py E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/mocks/dummydbm.py E302
|
||||
tests/test_cmdline/__init__.py E502 E501
|
||||
tests/test_cmdline/extensions.py E302
|
||||
tests/test_settings/__init__.py F401 E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501 E302
|
||||
tests/test_spiderloader/test_spiders/spider0.py E302
|
||||
tests/test_spiderloader/test_spiders/spider1.py E302
|
||||
tests/test_spiderloader/test_spiders/spider2.py E302
|
||||
tests/test_spiderloader/test_spiders/spider3.py E302
|
||||
tests/test_spiderloader/test_spiders/nested/spider4.py E302
|
||||
tests/test_utils_misc/__init__.py E501 E231
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
parsel>=1.5.0
|
||||
PyDispatcher>=2.0.5
|
||||
Twisted>=17.9.0
|
||||
w3lib>=1.17.0
|
||||
protego>=0.1.15
|
||||
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
|
||||
# Reference versions taken from
|
||||
# https://packages.ubuntu.com/xenial/python/
|
||||
# https://packages.ubuntu.com/xenial/zope/
|
||||
cssselect>=0.9.1
|
||||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
zope.interface>=4.1.3
|
||||
|
|
@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
|
|||
del warnings
|
||||
|
||||
# Apply monkey patches to fix issues in external libraries
|
||||
from . import _monkeypatches
|
||||
from scrapy import _monkeypatches
|
||||
del _monkeypatches
|
||||
|
||||
from twisted import version as _txv
|
||||
|
|
|
|||
|
|
@ -1,14 +1,4 @@
|
|||
import six
|
||||
from six.moves import copyreg
|
||||
|
||||
|
||||
if six.PY2:
|
||||
from urlparse import urlparse
|
||||
|
||||
# workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
|
||||
if urlparse('s3://bucket/key?key=value').query != 'key=value':
|
||||
from urlparse import uses_query
|
||||
uses_query.append('s3')
|
||||
import copyreg
|
||||
|
||||
|
||||
# Undo what Twisted's perspective broker adds to pickle register
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
|
@ -68,7 +67,7 @@ def _pop_command_name(argv):
|
|||
|
||||
def _print_header(settings, inproject):
|
||||
if inproject:
|
||||
print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
|
||||
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
|
||||
settings['BOT_NAME']))
|
||||
else:
|
||||
print("Scrapy %s - no active project\n" % scrapy.__version__)
|
||||
|
|
@ -124,7 +123,7 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
|
||||
from six.moves.urllib.parse import urlencode
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from __future__ import print_function
|
||||
import time
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from unittest import TextTestRunner, TextTestResult as _TextTestResult
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from __future__ import print_function
|
||||
import sys, six
|
||||
import sys
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -8,6 +7,7 @@ from scrapy.exceptions import UsageError
|
|||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
@ -24,12 +24,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true", \
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
@ -45,8 +44,7 @@ class Command(ScrapyCommand):
|
|||
self._print_bytes(response.body)
|
||||
|
||||
def _print_bytes(self, bytes_):
|
||||
bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
|
||||
bytes_writer.write(bytes_ + b'\n')
|
||||
sys.stdout.buffer.write(bytes_ + b'\n')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
import shutil
|
||||
import string
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import print_function
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import json
|
||||
import logging
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from __future__ import print_function
|
||||
import json
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from __future__ import print_function
|
||||
import re
|
||||
import os
|
||||
import string
|
||||
|
|
@ -44,8 +43,8 @@ class Command(ScrapyCommand):
|
|||
return False
|
||||
|
||||
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
|
||||
print('Error: Project names must begin with a letter and contain'\
|
||||
' only\nletters, numbers and underscores')
|
||||
print('Error: Project names must begin with a letter and contain'
|
||||
' only\nletters, numbers and underscores')
|
||||
elif _module_exists(project_name):
|
||||
print('Error: Module %r already exists' % project_name)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from __future__ import print_function
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.utils.versions import scrapy_components_versions
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from scrapy.commands import fetch, ScrapyCommand
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
||||
class Command(fetch.Command):
|
||||
|
||||
def short_desc(self):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from scrapy.item import BaseItem
|
|||
from scrapy.http import Request
|
||||
from scrapy.exceptions import ContractFail
|
||||
|
||||
from . import Contract
|
||||
from scrapy.contracts import Contract
|
||||
|
||||
|
||||
# contracts
|
||||
|
|
@ -86,8 +86,8 @@ class ReturnsContract(Contract):
|
|||
else:
|
||||
expected = '%s..%s' % (self.min_bound, self.max_bound)
|
||||
|
||||
raise ContractFail("Returned %s %s, expected %s" % \
|
||||
(occurrences, self.obj_name, expected))
|
||||
raise ContractFail("Returned %s %s, expected %s" %
|
||||
(occurrences, self.obj_name, expected))
|
||||
|
||||
|
||||
class ScrapesContract(Contract):
|
||||
|
|
|
|||
|
|
@ -1,19 +1,16 @@
|
|||
from __future__ import absolute_import
|
||||
import random
|
||||
import warnings
|
||||
from time import time
|
||||
from datetime import datetime
|
||||
from collections import deque
|
||||
|
||||
import six
|
||||
from twisted.internet import reactor, defer, task
|
||||
|
||||
from scrapy.utils.defer import mustbe_deferred
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.resolver import dnscache
|
||||
from scrapy import signals
|
||||
from .middleware import DownloaderMiddlewareManager
|
||||
from .handlers import DownloadHandlers
|
||||
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
|
||||
|
||||
class Slot(object):
|
||||
|
|
@ -190,7 +187,7 @@ class Downloader(object):
|
|||
|
||||
def close(self):
|
||||
self._slot_gc_loop.stop()
|
||||
for slot in six.itervalues(self.slots):
|
||||
for slot in self.slots.values():
|
||||
slot.close()
|
||||
|
||||
def _slot_gc(self, age=60):
|
||||
|
|
|
|||
|
|
@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
"""
|
||||
Twisted-recommended context factory for web clients.
|
||||
|
||||
Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
|
||||
|
||||
creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
|
||||
except this context factory allows setting the TLS/SSL method to use.
|
||||
The default is to use a
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
|
||||
have special requirements you can leave this as-is.
|
||||
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation.
|
||||
:meth:`creatorForNetloc` is the same as
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
|
||||
factory allows setting the TLS/SSL method to use.
|
||||
|
||||
The default OpenSSL method is ``TLS_METHOD`` (also called
|
||||
``SSLv23_METHOD``) which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
"""Download handlers for different schemes"""
|
||||
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
import six
|
||||
|
||||
from scrapy.exceptions import NotSupported, NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
|
|
@ -22,7 +23,7 @@ class DownloadHandlers(object):
|
|||
self._notconfigured = {} # remembers failed handlers
|
||||
handlers = without_none_values(
|
||||
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
|
||||
for scheme, clspath in six.iteritems(handlers):
|
||||
for scheme, clspath in handlers.items():
|
||||
self._schemes[scheme] = clspath
|
||||
self._load_handler(scheme, skip_lazy=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
|
|||
respcls = responsetypes.from_mimetype(uri.media_type)
|
||||
|
||||
resp_kwargs = {}
|
||||
if (issubclass(respcls, TextResponse) and
|
||||
uri.media_type.split('/')[0] == 'text'):
|
||||
if (issubclass(respcls, TextResponse)
|
||||
and uri.media_type.split('/')[0] == 'text'):
|
||||
charset = uri.media_type_parameters.get('charset')
|
||||
resp_kwargs['encoding'] = charset
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
|
|||
|
||||
import re
|
||||
from io import BytesIO
|
||||
from six.moves.urllib.parse import unquote
|
||||
from urllib.parse import unquote
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.protocols.ftp import FTPClient, CommandFailed
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from __future__ import absolute_import
|
||||
from .http10 import HTTP10DownloadHandler
|
||||
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http11 import (
|
||||
HTTP11DownloadHandler as HTTPDownloadHandler,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
|
||||
import re
|
||||
import logging
|
||||
import warnings
|
||||
from io import BytesIO
|
||||
from time import time
|
||||
import warnings
|
||||
from six.moves.urllib.parse import urldefrag
|
||||
from urllib.parse import urldefrag
|
||||
|
||||
from zope.interface import implementer
|
||||
from twisted.internet import defer, reactor, protocol
|
||||
|
|
@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss
|
|||
from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
|
||||
from twisted.internet.endpoints import TCP4ClientEndpoint
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.core.downloader.webclient import _parse
|
||||
|
|
@ -174,7 +175,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
|
|||
r"""
|
||||
Return binary content of a CONNECT request.
|
||||
|
||||
>>> from scrapy.utils.python import to_native_str as s
|
||||
>>> from scrapy.utils.python import to_unicode as s
|
||||
>>> s(tunnel_request_data("example.com", 8080))
|
||||
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
|
||||
>>> s(tunnel_request_data("example.com", 8080, b"123"))
|
||||
|
|
@ -285,6 +286,12 @@ class ScrapyAgent(object):
|
|||
scheme = _parse(request.url)[0]
|
||||
proxyHost = to_unicode(proxyHost)
|
||||
omitConnectTunnel = b'noconnect' in proxyParams
|
||||
if omitConnectTunnel:
|
||||
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
|
||||
"If you use Crawlera, it doesn't require this mode anymore, "
|
||||
"so you should update scrapy-crawlera to 1.3.0+ "
|
||||
"and remove '?noconnect' from the Crawlera URL.",
|
||||
ScrapyDeprecationWarning)
|
||||
if scheme == b'https' and not omitConnectTunnel:
|
||||
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
|
||||
proxyConf = (proxyHost, proxyPort, proxyAuth)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from six.moves.urllib.parse import unquote
|
||||
from urllib.parse import unquote
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.boto import is_botocore
|
||||
from .http import HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
||||
|
||||
|
||||
def _get_boto_connection():
|
||||
|
|
@ -21,7 +21,7 @@ def _get_boto_connection():
|
|||
return http_request.headers
|
||||
|
||||
try:
|
||||
import boto.auth
|
||||
import boto.auth # noqa: F401
|
||||
except ImportError:
|
||||
_S3Connection = _v19_S3Connection
|
||||
else:
|
||||
|
|
@ -32,8 +32,8 @@ def _get_boto_connection():
|
|||
|
||||
class S3DownloadHandler(object):
|
||||
|
||||
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
|
||||
httpdownloadhandler=HTTPDownloadHandler, **kw):
|
||||
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
|
||||
httpdownloadhandler=HTTPDownloadHandler, **kw):
|
||||
|
||||
if not aws_access_key_id:
|
||||
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
import six
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
|
|
@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
response = yield method(request=request, spider=spider)
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
|
||||
(six.get_method_self(method).__class__.__name__, response.__class__.__name__))
|
||||
(method.__self__.__class__.__name__, response.__class__.__name__))
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue((yield download_func(request=request, spider=spider)))
|
||||
|
|
@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
response = yield method(request=request, response=response, spider=spider)
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
|
||||
(six.get_method_self(method).__class__.__name__, type(response)))
|
||||
(method.__self__.__class__.__name__, type(response)))
|
||||
if isinstance(response, Request):
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(response)
|
||||
|
|
@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
response = yield method(request=request, exception=exception, spider=spider)
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
|
||||
(six.get_method_self(method).__class__.__name__, type(response)))
|
||||
(method.__self__.__class__.__name__, type(response)))
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(_failure)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from time import time
|
||||
from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
|
||||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.client import HTTPClientFactory
|
||||
from twisted.web.http import HTTPClient
|
||||
|
|
@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
delimiter = b'\n'
|
||||
|
||||
def connectionMade(self):
|
||||
self.headers = Headers() # bucket for response headers
|
||||
self.headers = Headers() # bucket for response headers
|
||||
|
||||
# Method command
|
||||
self.sendCommand(self.factory.method, self.factory.path)
|
||||
|
|
@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
if self.factory.url.startswith(b'https'):
|
||||
self.transport.stopProducing()
|
||||
|
||||
self.factory.noPage(\
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." % \
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
self.factory.noPage(
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." %
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class Slot(object):
|
|||
|
||||
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
|
||||
self.closing = False
|
||||
self.inprogress = set() # requests in progress
|
||||
self.inprogress = set() # requests in progress
|
||||
self.start_requests = iter(start_requests)
|
||||
self.close_if_idle = close_if_idle
|
||||
self.nextcall = nextcall
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ class Scraper(object):
|
|||
callback/errback"""
|
||||
assert isinstance(response, (Response, Failure))
|
||||
|
||||
dfd = self._scrape2(response, request, spider) # returns spiders processed output
|
||||
dfd = self._scrape2(response, request, spider) # returns spiders processed output
|
||||
dfd.addErrback(self.handle_spider_error, request, response, spider)
|
||||
dfd.addCallback(self.handle_spider_output, request, response, spider)
|
||||
return dfd
|
||||
|
|
@ -231,9 +231,9 @@ class Scraper(object):
|
|||
signal=signals.item_dropped, item=item, response=response,
|
||||
spider=spider, exception=output.value)
|
||||
else:
|
||||
logger.error('Error processing %(item)s', {'item': item},
|
||||
exc_info=failure_to_exc_info(output),
|
||||
extra={'spider': spider})
|
||||
logkws = self.logformatter.error(item, ex, response, spider)
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
|
||||
exc_info=failure_to_exc_info(output))
|
||||
return self.signals.send_catch_log_deferred(
|
||||
signal=signals.item_error, item=item, response=response,
|
||||
spider=spider, failure=output)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
|
|||
"""
|
||||
from itertools import chain, islice
|
||||
|
||||
import six
|
||||
from twisted.python.failure import Failure
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
|
|
@ -36,16 +35,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
|
||||
|
||||
def scrape_response(self, scrape_func, response, request, spider):
|
||||
fname = lambda f:'%s.%s' % (
|
||||
six.get_method_self(f).__class__.__name__,
|
||||
six.get_method_function(f).__name__)
|
||||
fname = lambda f: '%s.%s' % (
|
||||
f.__self__.__class__.__name__,
|
||||
f.__func__.__name__)
|
||||
|
||||
def process_spider_input(response):
|
||||
for method in self.methods['process_spider_input']:
|
||||
try:
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
|
||||
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
|
||||
.format(fname(method), type(result)))
|
||||
except _InvalidOutput:
|
||||
raise
|
||||
|
|
@ -70,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
elif result is None:
|
||||
continue
|
||||
else:
|
||||
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
|
||||
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
|
||||
.format(fname(method), type(result)))
|
||||
return _failure
|
||||
|
||||
|
|
@ -104,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
if _isiterable(result):
|
||||
result = evaluate_iterable(result, method_index)
|
||||
else:
|
||||
raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
|
||||
raise _InvalidOutput('Middleware {} must return an iterable, got {}'
|
||||
.format(fname(method), type(result)))
|
||||
|
||||
return chain(result, recovered)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import six
|
||||
import signal
|
||||
import logging
|
||||
import pprint
|
||||
import signal
|
||||
import warnings
|
||||
|
||||
import sys
|
||||
from twisted.internet import reactor, defer
|
||||
from zope.interface.verify import verifyClass, DoesNotImplement
|
||||
|
||||
|
|
@ -22,6 +21,7 @@ from scrapy.utils.log import (
|
|||
get_scrapy_root_handler, install_scrapy_root_handler)
|
||||
from scrapy import signals
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -46,7 +46,8 @@ class Crawler(object):
|
|||
logging.root.addHandler(handler)
|
||||
|
||||
d = dict(overridden_settings(self.settings))
|
||||
logger.info("Overridden settings: %(settings)r", {'settings': d})
|
||||
logger.info("Overridden settings:\n%(settings)s",
|
||||
{'settings': pprint.pformat(d)})
|
||||
|
||||
if get_scrapy_root_handler() is not None:
|
||||
# scrapy root handler already installed: update it with new settings
|
||||
|
|
@ -88,20 +89,9 @@ class Crawler(object):
|
|||
yield self.engine.open_spider(self.spider, start_requests)
|
||||
yield defer.maybeDeferred(self.engine.start)
|
||||
except Exception:
|
||||
# In Python 2 reraising an exception after yield discards
|
||||
# the original traceback (see https://bugs.python.org/issue7563),
|
||||
# so sys.exc_info() workaround is used.
|
||||
# This workaround also works in Python 3, but it is not needed,
|
||||
# and it is slower, so in Python 3 we use native `raise`.
|
||||
if six.PY2:
|
||||
exc_info = sys.exc_info()
|
||||
|
||||
self.crawling = False
|
||||
if self.engine is not None:
|
||||
yield self.engine.close()
|
||||
|
||||
if six.PY2:
|
||||
six.reraise(*exc_info)
|
||||
raise
|
||||
|
||||
def _create_spider(self, *args, **kwargs):
|
||||
|
|
@ -122,7 +112,7 @@ class Crawler(object):
|
|||
class CrawlerRunner(object):
|
||||
"""
|
||||
This is a convenient helper class that keeps track of, manages and runs
|
||||
crawlers inside an already setup Twisted `reactor`_.
|
||||
crawlers inside an already setup :mod:`~twisted.internet.reactor`.
|
||||
|
||||
The CrawlerRunner object must be instantiated with a
|
||||
:class:`~scrapy.settings.Settings` object.
|
||||
|
|
@ -216,7 +206,7 @@ class CrawlerRunner(object):
|
|||
return self._create_crawler(crawler_or_spidercls)
|
||||
|
||||
def _create_crawler(self, spidercls):
|
||||
if isinstance(spidercls, six.string_types):
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings)
|
||||
|
||||
|
|
@ -245,12 +235,13 @@ class CrawlerProcess(CrawlerRunner):
|
|||
A class to run multiple scrapy crawlers in a process simultaneously.
|
||||
|
||||
This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
|
||||
for starting a Twisted `reactor`_ and handling shutdown signals, like the
|
||||
keyboard interrupt command Ctrl-C. It also configures top-level logging.
|
||||
for starting a :mod:`~twisted.internet.reactor` and handling shutdown
|
||||
signals, like the keyboard interrupt command Ctrl-C. It also configures
|
||||
top-level logging.
|
||||
|
||||
This utility should be a better fit than
|
||||
:class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
|
||||
Twisted `reactor`_ within your application.
|
||||
:mod:`~twisted.internet.reactor` within your application.
|
||||
|
||||
The CrawlerProcess object must be instantiated with a
|
||||
:class:`~scrapy.settings.Settings` object.
|
||||
|
|
@ -285,9 +276,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def start(self, stop_after_crawl=True):
|
||||
"""
|
||||
This method starts a Twisted `reactor`_, adjusts its pool size to
|
||||
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
|
||||
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import absolute_import
|
||||
import re
|
||||
import logging
|
||||
|
||||
import six
|
||||
from w3lib import html
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -67,7 +65,9 @@ class AjaxCrawlMiddleware(object):
|
|||
|
||||
|
||||
# XXX: move it to w3lib?
|
||||
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
|
||||
_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
|
||||
|
||||
|
||||
def _has_ajaxcrawlable_meta(text):
|
||||
"""
|
||||
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import os
|
||||
import six
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -53,7 +52,7 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _debug_cookie(self, request, spider):
|
||||
if self.debug:
|
||||
cl = [to_native_str(c, errors='replace')
|
||||
cl = [to_unicode(c, errors='replace')
|
||||
for c in request.headers.getlist('Cookie')]
|
||||
if cl:
|
||||
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
|
||||
|
|
@ -62,7 +61,7 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _debug_set_cookie(self, response, spider):
|
||||
if self.debug:
|
||||
cl = [to_native_str(c, errors='replace')
|
||||
cl = [to_unicode(c, errors='replace')
|
||||
for c in response.headers.getlist('Set-Cookie')]
|
||||
if cl:
|
||||
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
|
||||
|
|
@ -82,8 +81,10 @@ class CookiesMiddleware(object):
|
|||
|
||||
def _get_request_cookies(self, jar, request):
|
||||
if isinstance(request.cookies, dict):
|
||||
cookie_list = [{'name': k, 'value': v} for k, v in \
|
||||
six.iteritems(request.cookies)]
|
||||
cookie_list = [
|
||||
{'name': k, 'value': v}
|
||||
for k, v in request.cookies.items()
|
||||
]
|
||||
else:
|
||||
cookie_list = request.cookies
|
||||
|
||||
|
|
|
|||
|
|
@ -4,20 +4,15 @@ and extract the potentially compressed responses that may arrive.
|
|||
|
||||
import bz2
|
||||
import gzip
|
||||
import zipfile
|
||||
import tarfile
|
||||
import logging
|
||||
import tarfile
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
from tempfile import mktemp
|
||||
|
||||
import six
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -79,7 +74,7 @@ class DecompressionMiddleware(object):
|
|||
if not response.body:
|
||||
return response
|
||||
|
||||
for fmt, func in six.iteritems(self._formats):
|
||||
for fmt, func in self._formats.items():
|
||||
new_response = func(response)
|
||||
if new_response:
|
||||
logger.debug('Decompressed response with format: %(responsefmt)s',
|
||||
|
|
|
|||
|
|
@ -1,11 +1,19 @@
|
|||
from email.utils import formatdate
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
|
|||
if content_encoding:
|
||||
encoding = content_encoding.pop()
|
||||
decoded_body = self._decode(response.body, encoding.lower())
|
||||
respcls = responsetypes.from_args(headers=response.headers, \
|
||||
url=response.url, body=decoded_body)
|
||||
respcls = responsetypes.from_args(
|
||||
headers=response.headers, url=response.url, body=decoded_body
|
||||
)
|
||||
kwargs = dict(cls=respcls, body=decoded_body)
|
||||
if issubclass(respcls, TextResponse):
|
||||
# force recalculating the encoding until we make sure the
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import base64
|
||||
from six.moves.urllib.parse import unquote, urlunparse
|
||||
from six.moves.urllib.request import getproxies, proxy_bypass
|
||||
try:
|
||||
from urllib2 import _parse_proxy
|
||||
except ImportError:
|
||||
from urllib.request import _parse_proxy
|
||||
from urllib.parse import unquote, urlunparse
|
||||
from urllib.request import getproxies, proxy_bypass, _parse_proxy
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
from six.moves.urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
|
|
@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
|
|||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
|
|||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import re
|
||||
|
||||
from twisted.internet.defer import Deferred, maybeDeferred
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
import logging
|
||||
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
|
||||
|
||||
class BaseDupeFilter(object):
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ new exceptions here without documenting them there.
|
|||
|
||||
# Internal
|
||||
|
||||
|
||||
class NotConfigured(Exception):
|
||||
"""Indicates a missing configuration situation"""
|
||||
pass
|
||||
|
||||
|
||||
class _InvalidOutput(TypeError):
|
||||
"""
|
||||
Indicates an invalid value has been returned by a middleware's processing method.
|
||||
|
|
@ -18,15 +20,19 @@ class _InvalidOutput(TypeError):
|
|||
"""
|
||||
pass
|
||||
|
||||
|
||||
# HTTP and crawling
|
||||
|
||||
|
||||
class IgnoreRequest(Exception):
|
||||
"""Indicates a decision was made not to process a request"""
|
||||
|
||||
|
||||
class DontCloseSpider(Exception):
|
||||
"""Request the spider not to be closed yet"""
|
||||
pass
|
||||
|
||||
|
||||
class CloseSpider(Exception):
|
||||
"""Raise this from callbacks to request the spider to be closed"""
|
||||
|
||||
|
|
@ -34,30 +40,37 @@ class CloseSpider(Exception):
|
|||
super(CloseSpider, self).__init__()
|
||||
self.reason = reason
|
||||
|
||||
|
||||
# Items
|
||||
|
||||
|
||||
class DropItem(Exception):
|
||||
"""Drop item from the item pipeline"""
|
||||
pass
|
||||
|
||||
|
||||
class NotSupported(Exception):
|
||||
"""Indicates a feature or method is not supported"""
|
||||
pass
|
||||
|
||||
|
||||
# Commands
|
||||
|
||||
|
||||
class UsageError(Exception):
|
||||
"""To indicate a command-line usage error"""
|
||||
def __init__(self, *a, **kw):
|
||||
self.print_help = kw.pop('print_help', True)
|
||||
super(UsageError, self).__init__(*a, **kw)
|
||||
|
||||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
"""Warning category for deprecated features, since the default
|
||||
DeprecationWarning is silenced on Python 2.7+
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class ContractFail(AssertionError):
|
||||
"""Error raised in case of a failing contract"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -4,18 +4,16 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
import pprint
|
||||
import marshal
|
||||
import six
|
||||
from six.moves import cPickle as pickle
|
||||
import warnings
|
||||
import pickle
|
||||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
|
||||
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
import warnings
|
||||
|
||||
|
||||
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
|
||||
|
|
@ -31,7 +29,7 @@ class BaseItemExporter(object):
|
|||
def _configure(self, options, dont_fail=False):
|
||||
"""Configure the exporter by poping options from the ``options`` dict.
|
||||
If dont_fail is set, it won't raise an exception on unexpected options
|
||||
(useful for using with keyword arguments in subclasses constructors)
|
||||
(useful for using with keyword arguments in subclasses ``__init__`` methods)
|
||||
"""
|
||||
self.encoding = options.pop('encoding', None)
|
||||
self.fields_to_export = options.pop('fields_to_export', None)
|
||||
|
|
@ -61,9 +59,9 @@ class BaseItemExporter(object):
|
|||
include_empty = self.export_empty_fields
|
||||
if self.fields_to_export is None:
|
||||
if include_empty and not isinstance(item, dict):
|
||||
field_iter = six.iterkeys(item.fields)
|
||||
field_iter = item.fields.keys()
|
||||
else:
|
||||
field_iter = six.iterkeys(item)
|
||||
field_iter = item.keys()
|
||||
else:
|
||||
if include_empty:
|
||||
field_iter = self.fields_to_export
|
||||
|
|
@ -143,11 +141,11 @@ class XmlItemExporter(BaseItemExporter):
|
|||
|
||||
def _beautify_newline(self, new_item=False):
|
||||
if self.indent is not None and (self.indent > 0 or new_item):
|
||||
self._xg_characters('\n')
|
||||
self.xg.characters('\n')
|
||||
|
||||
def _beautify_indent(self, depth=1):
|
||||
if self.indent:
|
||||
self._xg_characters(' ' * self.indent * depth)
|
||||
self.xg.characters(' ' * self.indent * depth)
|
||||
|
||||
def start_exporting(self):
|
||||
self.xg.startDocument()
|
||||
|
|
@ -181,27 +179,13 @@ class XmlItemExporter(BaseItemExporter):
|
|||
for value in serialized_value:
|
||||
self._export_xml_field('value', value, depth=depth+1)
|
||||
self._beautify_indent(depth=depth)
|
||||
elif isinstance(serialized_value, six.text_type):
|
||||
self._xg_characters(serialized_value)
|
||||
elif isinstance(serialized_value, str):
|
||||
self.xg.characters(serialized_value)
|
||||
else:
|
||||
self._xg_characters(str(serialized_value))
|
||||
self.xg.characters(str(serialized_value))
|
||||
self.xg.endElement(name)
|
||||
self._beautify_newline()
|
||||
|
||||
# Workaround for https://bugs.python.org/issue17606
|
||||
# Before Python 2.7.4 xml.sax.saxutils required bytes;
|
||||
# since 2.7.4 it requires unicode. The bug is likely to be
|
||||
# fixed in 2.7.6, but 2.7.6 will still support unicode,
|
||||
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
|
||||
if sys.version_info[:3] >= (2, 7, 4):
|
||||
def _xg_characters(self, serialized_value):
|
||||
if not isinstance(serialized_value, six.text_type):
|
||||
serialized_value = serialized_value.decode(self.encoding)
|
||||
return self.xg.characters(serialized_value)
|
||||
else: # pragma: no cover
|
||||
def _xg_characters(self, serialized_value):
|
||||
return self.xg.characters(serialized_value)
|
||||
|
||||
|
||||
class CsvItemExporter(BaseItemExporter):
|
||||
|
||||
|
|
@ -215,8 +199,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
line_buffering=False,
|
||||
write_through=True,
|
||||
encoding=self.encoding,
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
) if six.PY3 else file
|
||||
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
|
||||
)
|
||||
self.csv_writer = csv.writer(self.stream, **kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
|
|
@ -246,7 +230,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def _build_row(self, values):
|
||||
for s in values:
|
||||
try:
|
||||
yield to_native_str(s, self.encoding)
|
||||
yield to_unicode(s, self.encoding)
|
||||
except TypeError:
|
||||
yield s
|
||||
|
||||
|
|
@ -334,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
|
|||
if is_listlike(value):
|
||||
return [self._serialize_value(v) for v in value]
|
||||
encode_func = to_bytes if self.binary else to_unicode
|
||||
if isinstance(value, (six.text_type, bytes)):
|
||||
if isinstance(value, (str, bytes)):
|
||||
return encode_func(value, encoding=self.encoding)
|
||||
return value
|
||||
|
||||
def _serialize_dict(self, value):
|
||||
for key, val in six.iteritems(value):
|
||||
for key, val in value.items():
|
||||
key = to_bytes(key) if self.binary else key
|
||||
yield key, self._serialize_value(val)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
|
|||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.utils.conf import build_component_list
|
||||
|
||||
|
||||
class ExtensionManager(MiddlewareManager):
|
||||
|
||||
component_name = 'extension'
|
||||
|
|
|
|||
|
|
@ -54,9 +54,9 @@ class CloseSpider(object):
|
|||
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.task = reactor.callLater(self.close_on['timeout'], \
|
||||
self.crawler.engine.close_spider, spider, \
|
||||
reason='closespider_timeout')
|
||||
self.task = reactor.callLater(self.close_on['timeout'],
|
||||
self.crawler.engine.close_spider, spider,
|
||||
reason='closespider_timeout')
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
self.counter['itemcount'] += 1
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ import logging
|
|||
import posixpath
|
||||
from tempfile import NamedTemporaryFile
|
||||
from datetime import datetime
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse, unquote
|
||||
from urllib.parse import urlparse, unquote
|
||||
from ftplib import FTP
|
||||
|
||||
from zope.interface import Interface, implementer
|
||||
|
|
@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
|
|||
|
||||
def __init__(self, uri, _stdout=None):
|
||||
if not _stdout:
|
||||
_stdout = sys.stdout if six.PY2 else sys.stdout.buffer
|
||||
_stdout = sys.stdout.buffer
|
||||
self._stdout = _stdout
|
||||
|
||||
def open(self, spider):
|
||||
|
|
@ -199,9 +198,9 @@ class FeedExporter(object):
|
|||
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.urifmt = settings['FEED_URI']
|
||||
if not self.urifmt:
|
||||
if not settings['FEED_URI']:
|
||||
raise NotConfigured
|
||||
self.urifmt = str(settings['FEED_URI'])
|
||||
self.format = settings['FEED_FORMAT'].lower()
|
||||
self.export_encoding = settings['FEED_EXPORT_ENCODING']
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
|
|
|
|||
|
|
@ -1,23 +1,19 @@
|
|||
from __future__ import print_function
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
import os
|
||||
import pickle
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from importlib import import_module
|
||||
from time import time
|
||||
from warnings import warn
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from six.moves import cPickle as pickle
|
||||
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
|
|||
"""
|
||||
|
||||
import gc
|
||||
import six
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -28,7 +27,7 @@ class MemoryDebugger(object):
|
|||
def spider_closed(self, spider, reason):
|
||||
gc.collect()
|
||||
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
|
||||
for cls, wdict in six.iteritems(live_refs):
|
||||
for cls, wdict in live_refs.items():
|
||||
if not wdict:
|
||||
continue
|
||||
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import os
|
||||
from six.moves import cPickle as pickle
|
||||
import pickle
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.job import job_dir
|
||||
|
||||
|
||||
class SpiderState(object):
|
||||
"""Store and load spider state during a scraping job"""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import time
|
||||
from six.moves.http_cookiejar import (
|
||||
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
|
||||
)
|
||||
from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
|
||||
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
class CookieJar(object):
|
||||
|
|
@ -165,13 +164,13 @@ class WrappedRequest(object):
|
|||
return name in self.request.headers
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return to_native_str(self.request.headers.get(name, default),
|
||||
errors='replace')
|
||||
return to_unicode(self.request.headers.get(name, default),
|
||||
errors='replace')
|
||||
|
||||
def header_items(self):
|
||||
return [
|
||||
(to_native_str(k, errors='replace'),
|
||||
[to_native_str(x, errors='replace') for x in v])
|
||||
(to_unicode(k, errors='replace'),
|
||||
[to_unicode(x, errors='replace') for x in v])
|
||||
for k, v in self.request.headers.items()
|
||||
]
|
||||
|
||||
|
|
@ -189,7 +188,7 @@ class WrappedResponse(object):
|
|||
|
||||
# python3 cookiejars calls get_all
|
||||
def get_all(self, name, default=None):
|
||||
return [to_native_str(v, errors='replace')
|
||||
return [to_unicode(v, errors='replace')
|
||||
for v in self.response.headers.getlist(name)]
|
||||
# python2 cookiejars calls getheaders
|
||||
getheaders = get_all
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import six
|
||||
from w3lib.http import headers_dict_to_raw
|
||||
from scrapy.utils.datatypes import CaselessDict
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -19,7 +18,7 @@ class Headers(CaselessDict):
|
|||
"""Normalize values to bytes"""
|
||||
if value is None:
|
||||
value = []
|
||||
elif isinstance(value, (six.text_type, bytes)):
|
||||
elif isinstance(value, (str, bytes)):
|
||||
value = [value]
|
||||
elif not hasattr(value, '__iter__'):
|
||||
value = [value]
|
||||
|
|
@ -29,10 +28,10 @@ class Headers(CaselessDict):
|
|||
def _tobytes(self, x):
|
||||
if isinstance(x, bytes):
|
||||
return x
|
||||
elif isinstance(x, six.text_type):
|
||||
elif isinstance(x, str):
|
||||
return x.encode(self.encoding)
|
||||
elif isinstance(x, int):
|
||||
return six.text_type(x).encode(self.encoding)
|
||||
return str(x).encode(self.encoding)
|
||||
else:
|
||||
raise TypeError('Unsupported value type: {}'.format(type(x)))
|
||||
|
||||
|
|
@ -68,9 +67,6 @@ class Headers(CaselessDict):
|
|||
self[key] = lst
|
||||
|
||||
def items(self):
|
||||
return list(self.iteritems())
|
||||
|
||||
def iteritems(self):
|
||||
return ((k, self.getlist(k)) for k in self.keys())
|
||||
|
||||
def values(self):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ requests in Scrapy.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
import six
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.http.headers import Headers
|
||||
|
|
@ -60,13 +59,13 @@ class Request(object_ref):
|
|||
return self._url
|
||||
|
||||
def _set_url(self, url):
|
||||
if not isinstance(url, six.string_types):
|
||||
if not isinstance(url, str):
|
||||
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
|
||||
|
||||
s = safe_url_string(url, self.encoding)
|
||||
self._url = escape_ajax(s)
|
||||
|
||||
if ':' not in self._url:
|
||||
if ('://' not in self._url) and (not self._url.startswith('data:')):
|
||||
raise ValueError('Missing scheme in request url: %s' % self._url)
|
||||
|
||||
url = property(_get_url, obsolete_setter(_set_url, 'url'))
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
|
|||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin, urlencode
|
||||
from urllib.parse import urljoin, urlencode
|
||||
|
||||
import lxml.html
|
||||
from parsel.selector import create_root_node
|
||||
|
|
@ -104,8 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
|
|||
el = el.getparent()
|
||||
if el is None:
|
||||
break
|
||||
encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
|
||||
raise ValueError('No <form> element found with %s' % encoded)
|
||||
raise ValueError('No <form> element found with %s' % formxpath)
|
||||
|
||||
# If we get here, it means that either formname was None
|
||||
# or invalid
|
||||
|
|
@ -209,7 +207,7 @@ def _get_clickable(clickdata, form):
|
|||
# We didn't find it, so now we build an XPath expression out of the other
|
||||
# arguments, because they can be used as such
|
||||
xpath = u'.//*' + \
|
||||
u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
|
||||
u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
|
||||
el = form.xpath(xpath)
|
||||
if len(el) == 1:
|
||||
return (el[0].get('name'), el[0].get('value') or '')
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
from six.moves import xmlrpc_client as xmlrpclib
|
||||
import xmlrpc.client as xmlrpclib
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ responses in Scrapy.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.headers import Headers
|
||||
|
|
@ -88,7 +88,7 @@ class Response(object_ref):
|
|||
@property
|
||||
def text(self):
|
||||
"""For subclasses of TextResponse, this will return the body
|
||||
as text (unicode object in Python 2 and str in Python 3)
|
||||
as str
|
||||
"""
|
||||
raise AttributeError("Response content isn't text")
|
||||
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class HtmlResponse(TextResponse):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
|
|||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import parsel
|
||||
from w3lib.encoding import html_to_unicode, resolve_encoding, \
|
||||
|
|
@ -16,7 +15,7 @@ from w3lib.html import strip_html5_whitespace
|
|||
from scrapy.http.request import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_native_str
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
||||
|
||||
class TextResponse(Response):
|
||||
|
|
@ -31,17 +30,14 @@ class TextResponse(Response):
|
|||
super(TextResponse, self).__init__(*args, **kwargs)
|
||||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, six.text_type):
|
||||
if six.PY2 and self.encoding is None:
|
||||
raise TypeError("Cannot convert unicode url - %s "
|
||||
"has no encoding" % type(self).__name__)
|
||||
self._url = to_native_str(url, self.encoding)
|
||||
if isinstance(url, str):
|
||||
self._url = to_unicode(url, self.encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_url(url)
|
||||
|
||||
def _set_body(self, body):
|
||||
self._body = b'' # used by encoding detection
|
||||
if isinstance(body, six.text_type):
|
||||
if isinstance(body, str):
|
||||
if self._encoding is None:
|
||||
raise TypeError('Cannot convert unicode body - %s has no encoding' %
|
||||
type(self).__name__)
|
||||
|
|
@ -84,11 +80,11 @@ class TextResponse(Response):
|
|||
@memoizemethod_noargs
|
||||
def _headers_encoding(self):
|
||||
content_type = self.headers.get(b'Content-Type', b'')
|
||||
return http_content_type_encoding(to_native_str(content_type))
|
||||
return http_content_type_encoding(to_unicode(content_type))
|
||||
|
||||
def _body_inferred_encoding(self):
|
||||
if self._cached_benc is None:
|
||||
content_type = to_native_str(self.headers.get(b'Content-Type', b''))
|
||||
content_type = to_unicode(self.headers.get(b'Content-Type', b''))
|
||||
benc, ubody = html_to_unicode(content_type, self.body,
|
||||
auto_detect_fun=self._auto_detect_fun,
|
||||
default_encoding=self._DEFAULT_ENCODING)
|
||||
|
|
@ -161,7 +157,7 @@ class TextResponse(Response):
|
|||
|
||||
def _url_from_selector(sel):
|
||||
# type: (parsel.Selector) -> str
|
||||
if isinstance(sel.root, six.string_types):
|
||||
if isinstance(sel.root, str):
|
||||
# e.g. ::attr(href) result
|
||||
return strip_html5_whitespace(sel.root)
|
||||
if not hasattr(sel.root, 'tag'):
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class XmlResponse(TextResponse):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from zope.interface import Interface
|
||||
|
||||
|
||||
class ISpiderLoader(Interface):
|
||||
|
||||
def from_settings(settings):
|
||||
|
|
|
|||
|
|
@ -4,24 +4,16 @@ Scrapy Item
|
|||
See documentation in docs/topics/item.rst
|
||||
"""
|
||||
|
||||
import collections
|
||||
from abc import ABCMeta
|
||||
from collections.abc import MutableMapping
|
||||
from copy import deepcopy
|
||||
from pprint import pformat
|
||||
from warnings import warn
|
||||
|
||||
import six
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
if six.PY2:
|
||||
MutableMapping = collections.MutableMapping
|
||||
else:
|
||||
MutableMapping = collections.abc.MutableMapping
|
||||
|
||||
|
||||
class BaseItem(object_ref):
|
||||
"""Base class for all scraped items.
|
||||
|
||||
|
|
@ -84,7 +76,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
if args or kwargs: # avoid creating dict for most common case
|
||||
for k, v in six.iteritems(dict(*args, **kwargs)):
|
||||
for k, v in dict(*args, **kwargs).items():
|
||||
self[k] = v
|
||||
|
||||
def __getitem__(self, key):
|
||||
|
|
@ -136,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
|
|||
return deepcopy(self)
|
||||
|
||||
|
||||
@six.add_metaclass(ItemMeta)
|
||||
class Item(DictItem):
|
||||
class Item(DictItem, metaclass=ItemMeta):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -4,10 +4,6 @@ This module defines the Link object used in Link extractors.
|
|||
For actual link extractors implementation see scrapy.linkextractors, or
|
||||
its documentation in: docs/topics/link-extractors.rst
|
||||
"""
|
||||
import warnings
|
||||
import six
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
||||
class Link(object):
|
||||
|
|
@ -17,13 +13,8 @@ class Link(object):
|
|||
|
||||
def __init__(self, url, text='', fragment='', nofollow=False):
|
||||
if not isinstance(url, str):
|
||||
if six.PY2:
|
||||
warnings.warn("Link urls must be str objects. "
|
||||
"Assuming utf-8 encoding (which could be wrong)")
|
||||
url = to_bytes(url, encoding='utf8')
|
||||
else:
|
||||
got = url.__class__.__name__
|
||||
raise TypeError("Link urls must be str objects, got %s" % got)
|
||||
got = url.__class__.__name__
|
||||
raise TypeError("Link urls must be str objects, got %s" % got)
|
||||
self.url = url
|
||||
self.text = text
|
||||
self.fragment = fragment
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
|
|||
For more info see docs/topics/link-extractors.rst
|
||||
"""
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from six.moves.urllib.parse import urlparse
|
||||
from parsel.csstranslator import HTMLTranslator
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
|
|
@ -19,30 +19,32 @@ from scrapy.utils.url import (
|
|||
|
||||
# common file extensions that are not followed if they occur in links
|
||||
IGNORED_EXTENSIONS = [
|
||||
# archives
|
||||
'7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip',
|
||||
|
||||
# images
|
||||
'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
|
||||
'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg',
|
||||
'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico',
|
||||
|
||||
# audio
|
||||
'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff',
|
||||
|
||||
# video
|
||||
'3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
|
||||
'm4a', 'm4v', 'flv',
|
||||
'm4a', 'm4v', 'flv', 'webm',
|
||||
|
||||
# office suites
|
||||
'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',
|
||||
'odp',
|
||||
|
||||
# other
|
||||
'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar',
|
||||
'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk'
|
||||
]
|
||||
|
||||
|
||||
_re_type = type(re.compile("", 0))
|
||||
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \
|
||||
'file', 'ftp'}
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
|
||||
|
||||
|
||||
class FilteringLinkExtractor(object):
|
||||
|
|
@ -115,4 +117,4 @@ class FilteringLinkExtractor(object):
|
|||
|
||||
|
||||
# Top-level imports
|
||||
from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
|
||||
|
|
|
|||
|
|
@ -2,9 +2,8 @@
|
|||
HTMLParser-based link extractor
|
||||
"""
|
||||
import warnings
|
||||
import six
|
||||
from six.moves.html_parser import HTMLParser
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
|
@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser):
|
|||
ret = []
|
||||
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in links:
|
||||
if isinstance(link.url, six.text_type):
|
||||
if isinstance(link.url, str):
|
||||
link.url = link.url.encode(response_encoding)
|
||||
try:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
"""
|
||||
Link extractor based on lxml.html
|
||||
"""
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import lxml.etree as etree
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
|
@ -10,7 +9,7 @@ from w3lib.url import canonicalize_url
|
|||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list, to_native_str
|
||||
from scrapy.utils.python import unique as unique_list, to_unicode
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
|
||||
|
|
@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()")
|
|||
|
||||
|
||||
def _nons(tag):
|
||||
if isinstance(tag, six.string_types):
|
||||
if isinstance(tag, str):
|
||||
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
|
||||
return tag.split('}')[-1]
|
||||
return tag
|
||||
|
|
@ -67,7 +66,7 @@ class LxmlParserLinkExtractor(object):
|
|||
url = self.process_attr(attr_val)
|
||||
if url is None:
|
||||
continue
|
||||
url = to_native_str(url, encoding=response_encoding)
|
||||
url = to_unicode(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import re
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
|
||||
|
||||
from scrapy.link import Link
|
||||
from .sgml import SgmlLinkExtractor
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor
|
||||
|
||||
|
||||
linkre = re.compile(
|
||||
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
"""
|
||||
SGMLParser-based Link extractors
|
||||
"""
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin
|
||||
import warnings
|
||||
from urllib.parse import urljoin
|
||||
from sgmllib import SGMLParser
|
||||
|
||||
from w3lib.url import safe_url_string, canonicalize_url
|
||||
|
|
@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
|
|||
if base_url is None:
|
||||
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in self.links:
|
||||
if isinstance(link.url, six.text_type):
|
||||
if isinstance(link.url, str):
|
||||
link.url = link.url.encode(response_encoding)
|
||||
try:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ Item Loader
|
|||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from collections import defaultdict
|
||||
|
||||
import six
|
||||
from contextlib import suppress
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
|
|
@ -15,6 +14,17 @@ from scrapy.utils.misc import arg_to_iter, extract_regex
|
|||
from scrapy.utils.python import flatten
|
||||
|
||||
|
||||
def unbound_method(method):
|
||||
"""
|
||||
Allow to use single-argument functions as input or output processors
|
||||
(no need to define an unused first 'self' argument)
|
||||
"""
|
||||
with suppress(AttributeError):
|
||||
if '.' not in method.__qualname__:
|
||||
return method.__func__
|
||||
return method
|
||||
|
||||
|
||||
class ItemLoader(object):
|
||||
|
||||
default_item_class = Item
|
||||
|
|
@ -72,7 +82,7 @@ class ItemLoader(object):
|
|||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in six.iteritems(value):
|
||||
for k, v in value.items():
|
||||
self._add_value(k, v)
|
||||
else:
|
||||
self._add_value(field_name, value)
|
||||
|
|
@ -82,7 +92,7 @@ class ItemLoader(object):
|
|||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in six.iteritems(value):
|
||||
for k, v in value.items():
|
||||
self._replace_value(k, v)
|
||||
else:
|
||||
self._replace_value(field_name, value)
|
||||
|
|
@ -142,14 +152,14 @@ class ItemLoader(object):
|
|||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor',
|
||||
self.default_input_processor)
|
||||
return proc
|
||||
return unbound_method(proc)
|
||||
|
||||
def get_output_processor(self, field_name):
|
||||
proc = getattr(self, '%s_out' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor',
|
||||
self.default_output_processor)
|
||||
return proc
|
||||
return unbound_method(proc)
|
||||
|
||||
def _process_input_value(self, field_name, value):
|
||||
proc = self.get_input_processor(field_name)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from functools import partial
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
||||
|
||||
def wrap_loader_context(function, context):
|
||||
"""Wrap functions that receive loader_context to contain the context
|
||||
"pre-loaded" and expose a interface that receives only one argument
|
||||
|
|
|
|||
|
|
@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
try:
|
||||
from collections import ChainMap
|
||||
except ImportError:
|
||||
from scrapy.utils.datatypes import MergeDict as ChainMap
|
||||
from collections import ChainMap
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from scrapy.utils.request import referer_str
|
|||
SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
|
||||
DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
|
||||
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
|
||||
ERRORMSG = u"'Error processing %(item)s'"
|
||||
|
||||
|
||||
class LogFormatter(object):
|
||||
|
|
@ -92,6 +93,16 @@ class LogFormatter(object):
|
|||
}
|
||||
}
|
||||
|
||||
def error(self, item, exception, response, spider):
|
||||
"""Logs a message when an item causes an error while it is passing through the item pipeline."""
|
||||
return {
|
||||
'level': logging.ERROR,
|
||||
'msg': ERRORMSG,
|
||||
'args': {
|
||||
'item': item,
|
||||
}
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls()
|
||||
|
|
|
|||
|
|
@ -4,29 +4,20 @@ Mail sending helpers
|
|||
See documentation in docs/topics/email.rst
|
||||
"""
|
||||
import logging
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
from io import BytesIO
|
||||
import six
|
||||
|
||||
from email import encoders as Encoders
|
||||
from email.mime.base import MIMEBase
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.nonmultipart import MIMENonMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email.utils import COMMASPACE, formatdate
|
||||
from six.moves.email_mime_multipart import MIMEMultipart
|
||||
from six.moves.email_mime_text import MIMEText
|
||||
from six.moves.email_mime_base import MIMEBase
|
||||
if six.PY2:
|
||||
from email.MIMENonMultipart import MIMENonMultipart
|
||||
from email import Encoders
|
||||
else:
|
||||
from email.mime.nonmultipart import MIMENonMultipart
|
||||
from email import encoders as Encoders
|
||||
from io import BytesIO
|
||||
|
||||
from twisted.internet import defer, reactor, ssl
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -82,8 +73,7 @@ class MailSender(object):
|
|||
part = MIMEBase(*mimetype.split('/'))
|
||||
part.set_payload(f.read())
|
||||
Encoders.encode_base64(part)
|
||||
part.add_header('Content-Disposition', 'attachment; filename="%s"' \
|
||||
% attach_name)
|
||||
part.add_header('Content-Disposition', 'attachment', filename=attach_name)
|
||||
msg.attach(part)
|
||||
else:
|
||||
msg.set_payload(body)
|
||||
|
|
|
|||
|
|
@ -65,8 +65,8 @@ class MiddlewareManager(object):
|
|||
return process_chain(self.methods[methodname], obj, *args)
|
||||
|
||||
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
|
||||
return process_chain_both(self.methods[cb_methodname], \
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
return process_chain_both(self.methods[cb_methodname],
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
|
||||
def open_spider(self, spider):
|
||||
return self._process_parallel('open_spider', spider)
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue