RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'scrapy/master' into issue5043-feed_export

This commit is contained in:
Adrián Chaves 2023-02-02 05:56:48 +01:00
parent 2f2bcb006d c34ca4aef5
commit 32bc8bd436
345 changed files with 18541 additions and 13548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.bandit.yml
View File

@ -17,3 +17,4 @@ skips:
- B503
- B603
- B605
exclude_dirs: ['tests']

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.7.1
current_version = 2.8.0
commit = True
tag = True
tag_name = {new_version}

2
.flake8
View File

@ -1,7 +1,7 @@
[flake8]
max-line-length = 119
ignore = W503
ignore = W503, E203
exclude =
docs/conf.py

7
.git-blame-ignore-revs Normal file
View File

@ -0,0 +1,7 @@
# .git-blame-ignore-revs
# adding black formatter to all the code
e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
# re applying black to the code with default line length
303f0a70fcf8067adf0a909c2096a5009162383a
# reaplying black again and removing line length on pre-commit black config
c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962

14
.github/workflows/checks.yml vendored
View File

@ -9,14 +9,6 @@ jobs:
matrix:
include:
- python-version: "3.11"
env:
TOXENV: security
- python-version: "3.11"
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: pylint
- python-version: 3.7
@ -42,3 +34,9 @@ jobs:
run: |
pip install -U tox
tox
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: pre-commit/action@v3.0.0

4
.github/workflows/publish.yml vendored
View File

@ -24,8 +24,8 @@ jobs:
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
pip install --upgrade build twine
python -m build
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*

5
.github/workflows/tests-ubuntu.yml vendored
View File

@ -38,10 +38,7 @@ jobs:
env:
TOXENV: pypy3-pinned
# extras
# extra-deps includes reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
- python-version: "3.11"
env:
TOXENV: extra-deps

2
.isort.cfg Normal file
View File

@ -0,0 +1,2 @@
[settings]
profile = black

18
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,18 @@
repos:
- repo: https://github.com/PyCQA/bandit
rev: 1.7.4
hooks:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 6.0.0
hooks:
- id: flake8
- repo: https://github.com/psf/black.git
rev: 22.12.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort

22
conftest.py
View File

@ -4,12 +4,11 @@ import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
return (str(p) for p in Path(folder).rglob("*.py"))
collect_ignore = [
@ -21,16 +20,16 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
with open('tests/ignores.txt') as reader:
with Path("tests/ignores.txt").open(encoding="utf-8") as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
if file_path and file_path[0] != "#":
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
"scrapy/core/downloader/handlers/http2.py",
*_py_files("scrapy/core/http2"),
)
)
@ -50,7 +49,7 @@ def pytest_addoption(parser):
)
@pytest.fixture(scope='class')
@pytest.fixture(scope="class")
def reactor_pytest(request):
if not request.cls:
# doctests
@ -61,14 +60,17 @@ def reactor_pytest(request):
@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
pytest.skip("This test is only run with --reactor=asyncio")
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
pytest.skip('This test is only run without --reactor=asyncio')
if (
request.node.get_closest_marker("only_not_asyncio")
and reactor_pytest == "asyncio"
):
pytest.skip("This test is only run without --reactor=asyncio")
def pytest_configure(config):

4
docs/Makefile
View File

@ -86,8 +86,8 @@ coverage: BUILDER = coverage
coverage: build
htmlview: html
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
$(PYTHON) -c "import webbrowser; from pathlib import Path; \
webbrowser.open('file://' + Path('build/html/index.html').resolve())"
clean:
-rm -rf build/*

81
docs/_ext/scrapydocs.py
View File

@ -1,8 +1,9 @@
from docutils.parsers.rst.roles import set_classes
from operator import itemgetter
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst.roles import set_classes
from sphinx.util.nodes import make_refnode
from operator import itemgetter
class settingslist_node(nodes.General, nodes.Element):
@ -11,15 +12,15 @@ class settingslist_node(nodes.General, nodes.Element):
class SettingsListDirective(Directive):
def run(self):
return [settingslist_node('')]
return [settingslist_node("")]
def is_setting_index(node):
if node.tagname == 'index' and node['entries']:
if node.tagname == "index" and node["entries"]:
# index entries for setting directives look like:
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
entry_type, info, refid = node["entries"][0][:3]
return entry_type == "pair" and info.endswith("; setting")
return False
@ -30,14 +31,14 @@ def get_setting_target(node):
def get_setting_name_and_refid(node):
"""Extract setting name from directive index node"""
entry_type, info, refid = node['entries'][0][:3]
return info.replace('; setting', ''), refid
entry_type, info, refid = node["entries"][0][:3]
return info.replace("; setting", ""), refid
def collect_scrapy_settings_refs(app, doctree):
env = app.builder.env
if not hasattr(env, 'scrapy_all_settings'):
if not hasattr(env, "scrapy_all_settings"):
env.scrapy_all_settings = []
for node in doctree.traverse(is_setting_index):
@ -46,18 +47,23 @@ def collect_scrapy_settings_refs(app, doctree):
setting_name, refid = get_setting_name_and_refid(node)
env.scrapy_all_settings.append({
'docname': env.docname,
'setting_name': setting_name,
'refid': refid,
})
env.scrapy_all_settings.append(
{
"docname": env.docname,
"setting_name": setting_name,
"refid": refid,
}
)
def make_setting_element(setting_data, app, fromdocname):
refnode = make_refnode(app.builder, fromdocname,
todocname=setting_data['docname'],
targetid=setting_data['refid'],
child=nodes.Text(setting_data['setting_name']))
refnode = make_refnode(
app.builder,
fromdocname,
todocname=setting_data["docname"],
targetid=setting_data["refid"],
child=nodes.Text(setting_data["setting_name"]),
)
p = nodes.paragraph()
p += refnode
@ -71,10 +77,13 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
for node in doctree.traverse(settingslist_node):
settings_list = nodes.bullet_list()
settings_list.extend([make_setting_element(d, app, fromdocname)
for d in sorted(env.scrapy_all_settings,
key=itemgetter('setting_name'))
if fromdocname != d['docname']])
settings_list.extend(
[
make_setting_element(d, app, fromdocname)
for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name"))
if fromdocname != d["docname"]
]
)
node.replace_self(settings_list)
@ -99,41 +108,41 @@ def setup(app):
rolename="reqmeta",
indextemplate="pair: %s; reqmeta",
)
app.add_role('source', source_role)
app.add_role('commit', commit_role)
app.add_role('issue', issue_role)
app.add_role('rev', rev_role)
app.add_role("source", source_role)
app.add_role("commit", commit_role)
app.add_role("issue", issue_role)
app.add_role("rev", rev_role)
app.add_node(settingslist_node)
app.add_directive('settingslist', SettingsListDirective)
app.add_directive("settingslist", SettingsListDirective)
app.connect('doctree-read', collect_scrapy_settings_refs)
app.connect('doctree-resolved', replace_settingslist_nodes)
app.connect("doctree-read", collect_scrapy_settings_refs)
app.connect("doctree-resolved", replace_settingslist_nodes)
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
ref = "https://github.com/scrapy/scrapy/blob/master/" + text
set_classes(options)
node = nodes.reference(rawtext, text, refuri=ref, **options)
return [node], []
def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/issues/' + text
ref = "https://github.com/scrapy/scrapy/issues/" + text
set_classes(options)
node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options)
return [node], []
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/commit/' + text
ref = "https://github.com/scrapy/scrapy/commit/" + text
set_classes(options)
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options)
return [node], []
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
ref = "http://hg.scrapy.org/scrapy/changeset/" + text
set_classes(options)
node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "r" + text, refuri=ref, **options)
return [node], []

184
docs/conf.py
View File

@ -11,13 +11,12 @@
import sys
from datetime import datetime
from os import path
from pathlib import Path
# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
sys.path.append(path.join(path.dirname(__file__), "_ext"))
sys.path.insert(0, path.dirname(path.dirname(__file__)))
# is relative to the documentation root, use Path.absolute to make it absolute.
sys.path.append(str(Path(__file__).parent / "_ext"))
sys.path.insert(0, str(Path(__file__).parent.parent))
# General configuration
@ -26,30 +25,30 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'hoverxref.extension',
'notfound.extension',
'scrapydocs',
'sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
"hoverxref.extension",
"notfound.extension",
"scrapydocs",
"sphinx.ext.autodoc",
"sphinx.ext.coverage",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8'
# source_encoding = 'utf-8'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'Scrapy'
copyright = f'2008–{datetime.now().year}, Scrapy developers'
project = "Scrapy"
copyright = f"2008–{datetime.now().year}, Scrapy developers"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -58,50 +57,51 @@ copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The short X.Y version.
try:
import scrapy
version = '.'.join(map(str, scrapy.version_info[:2]))
version = ".".join(map(str, scrapy.version_info[:2]))
release = scrapy.__version__
except ImportError:
version = ''
release = ''
version = ""
release = ""
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
language = 'en'
language = "en"
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of documents that shouldn't be included in the build.
#unused_docs = []
# unused_docs = []
exclude_patterns = ['build']
exclude_patterns = ["build"]
# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['.build']
exclude_trees = [".build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# List of Sphinx warnings that will not be raised
suppress_warnings = ['epub.unknown_project_files']
suppress_warnings = ["epub.unknown_project_files"]
# Options for HTML output
@ -109,17 +109,18 @@ suppress_warnings = ['epub.unknown_project_files']
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
# Add path to the RTD explicitly to robustify builds (otherwise might
# fail in a clean Debian build env)
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
@ -129,44 +130,44 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'
html_last_updated_fmt = "%b %d, %Y"
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_use_modindex = True
# html_use_modindex = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, the reST sources are included in the HTML build as _sources/<name>.
html_copy_source = True
@ -174,16 +175,16 @@ html_copy_source = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''
# html_file_suffix = ''
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
htmlhelp_basename = "Scrapydoc"
html_css_files = [
'custom.css',
"custom.css",
]
@ -191,34 +192,33 @@ html_css_files = [
# ------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', 'Scrapy Documentation',
'Scrapy developers', 'manual'),
("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_use_modindex = True
# latex_use_modindex = True
# Options for the linkcheck builder
@ -227,8 +227,9 @@ latex_documents = [
# A list of regular expressions that match URIs that should not be checked when
# doing a linkcheck build.
linkcheck_ignore = [
'http://localhost:\d+', 'http://hg.scrapy.org',
'http://directory.google.com/'
"http://localhost:\d+",
"http://hg.scrapy.org",
"http://directory.google.com/",
]
@ -238,44 +239,35 @@ coverage_ignore_pyobjects = [
# Contract’s add_pre_hook and add_post_hook are not documented because
# they should be transparent to contract developers, for whom pre_hook and
# post_hook should be the actual concern.
r'\bContract\.add_(pre|post)_hook$',
r"\bContract\.add_(pre|post)_hook$",
# ContractsManager is an internal class, developers are not expected to
# interact with it directly in any way.
r'\bContractsManager\b$',
r"\bContractsManager\b$",
# For default contracts we only want to document their general purpose in
# their __init__ method, the methods they reimplement to achieve that purpose
# should be irrelevant to developers using those contracts.
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
r"\w+Contract\.(adjust_request_args|(pre|post)_process)$",
# Methods of downloader middlewares are not documented, only the classes
# themselves, since downloader middlewares are controlled through Scrapy
# settings.
r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
r"^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.",
# Base classes of downloader middlewares are implementation details that
# are not meant for users.
r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware",
# Private exception used by the command-line interface implementation.
r'^scrapy\.exceptions\.UsageError',
r"^scrapy\.exceptions\.UsageError",
# Methods of BaseItemExporter subclasses are only documented in
# BaseItemExporter.
r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
r"^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.",
# Extension behavior is only modified through settings. Methods of
# extension classes, as well as helper functions, are implementation
# details that are not documented.
r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
r"^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.", # methods
r"^scrapy\.extensions\.[a-z]\w*?\.[a-z]", # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
r"^scrapy\.linkextractors\.FilteringLinkExtractor$",
# Implementation detail of LxmlLinkExtractor
r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
]
@ -283,18 +275,18 @@ coverage_ignore_pyobjects = [
# -------------------------------------
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cryptography' : ('https://cryptography.io/en/latest/', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.wiki/en/latest/', None),
'twisted': ('https://docs.twisted.org/en/stable/', None),
'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
"attrs": ("https://www.attrs.org/en/stable/", None),
"coverage": ("https://coverage.readthedocs.io/en/stable", None),
"cryptography": ("https://cryptography.io/en/latest/", None),
"cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
"itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
"pytest": ("https://docs.pytest.org/en/latest", None),
"python": ("https://docs.python.org/3", None),
"sphinx": ("https://www.sphinx-doc.org/en/master", None),
"tox": ("https://tox.wiki/en/latest/", None),
"twisted": ("https://docs.twisted.org/en/stable/", None),
"twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
"w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
}
intersphinx_disabled_reftypes = []
@ -314,16 +306,16 @@ hoverxref_role_types = {
"setting": "tooltip",
"signal": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
def setup(app):
app.connect('autodoc-skip-member', maybe_skip_member)
app.connect("autodoc-skip-member", maybe_skip_member)
def maybe_skip_member(app, what, name, obj, skip, options):
if not skip:
# autodocs was generating a text "alias of" for the following members
# https://github.com/sphinx-doc/sphinx/issues/4422
return name in {'default_item_class', 'default_selector_class'}
return name in {"default_item_class", "default_selector_class"}
return skip

23
docs/conftest.py
View File

@ -1,33 +1,34 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
from scrapy.http.response.html import HtmlResponse
def load_response(url, filename):
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
with open(input_path, 'rb') as input_file:
return HtmlResponse(url, body=input_file.read())
def load_response(url: str, filename: str) -> HtmlResponse:
input_path = Path(__file__).parent / "_tests" / filename
return HtmlResponse(url, body=input_path.read_bytes())
def setup(namespace):
namespace['load_response'] = load_response
namespace["load_response"] = load_response
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
PythonCodeBlockParser(future_imports=['print_function']),
PythonCodeBlockParser(future_imports=["print_function"]),
skip,
],
pattern='*.rst',
pattern="*.rst",
setup=setup,
).pytest()

37
docs/contributing.rst
View File

@ -49,7 +49,7 @@ guidelines when you're going to report a new bug.
(use "scrapy" tag).
* check the `open issues`_ to see if the issue has already been reported. If it
has, don't dismiss the report, but check the ticket history and comments. If
has, don't dismiss the report, but check the ticket history and comments. If
you have additional useful information, please leave a comment, or consider
:ref:`sending a pull request <writing-patches>` with a fix.
@ -169,16 +169,43 @@ Coding style
Please follow these coding conventions when writing code for inclusion in
Scrapy:
* Unless otherwise specified, follow :pep:`8`.
* It's OK to use lines longer than 79 chars if it improves the code
readability.
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
run black manually with ``tox -e black``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
.. _scrapy-pre-commit:
Pre-commit
==========
We use `pre-commit`_ to automatically address simple code issues before every
commit.
.. _pre-commit: https://pre-commit.com/
After your create a local clone of your fork of the Scrapy repository:
#. `Install pre-commit <https://pre-commit.com/#installation>`_.
#. On the root of your local clone of the Scrapy repository, run the following
command:
.. code-block:: bash
pre-commit install
Now pre-commit will check your changes every time you create a Git commit. Upon
finding issues, pre-commit aborts your commit, and either fixes those issues
automatically, or only reports them to you. If it fixes those issues
automatically, creating your commit again should succeed. Otherwise, you may
need to address the corresponding issues manually first.
.. _documentation-policies:
Documentation policies

10
docs/intro/tutorial.rst
View File

@ -85,6 +85,8 @@ page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
from pathlib import Path
import scrapy
@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
self.log(f'Saved file {filename}')
@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
from pathlib import Path
import scrapy
@ -191,8 +194,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy

234
docs/news.rst
View File

@ -3,6 +3,212 @@
Release notes
=============
.. _release-2.8.0:
Scrapy 2.8.0 (2023-02-02)
-------------------------
This is a maintenance release, with minor features, bug fixes, and cleanups.
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- The ``scrapy.utils.gz.read1`` function, deprecated in Scrapy 2.0, has now
been removed. Use the :meth:`~io.BufferedIOBase.read1` method of
:class:`~gzip.GzipFile` instead.
(:issue:`5719`)
- The ``scrapy.utils.python.to_native_str`` function, deprecated in Scrapy
2.0, has now been removed. Use :func:`scrapy.utils.python.to_unicode`
instead.
(:issue:`5719`)
- The ``scrapy.utils.python.MutableChain.next`` method, deprecated in Scrapy
2.0, has now been removed. Use
:meth:`~scrapy.utils.python.MutableChain.__next__` instead.
(:issue:`5719`)
- The ``scrapy.linkextractors.FilteringLinkExtractor`` class, deprecated
in Scrapy 2.0, has now been removed. Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead.
(:issue:`5720`)
- Support for using environment variables prefixed with ``SCRAPY_`` to
override settings, deprecated in Scrapy 2.0, has now been removed.
(:issue:`5724`)
- Support for the ``noconnect`` query string argument in proxy URLs,
deprecated in Scrapy 2.0, has now been removed. We expect proxies that used
to need it to work fine without it.
(:issue:`5731`)
- The ``scrapy.utils.python.retry_on_eintr`` function, deprecated in Scrapy
2.3, has now been removed.
(:issue:`5719`)
- The ``scrapy.utils.python.WeakKeyCache`` class, deprecated in Scrapy 2.4,
has now been removed.
(:issue:`5719`)
Deprecations
~~~~~~~~~~~~
- :exc:`scrapy.pipelines.images.NoimagesDrop` is now deprecated.
(:issue:`5368`, :issue:`5489`)
- :meth:`ImagesPipeline.convert_image
<scrapy.pipelines.images.ImagesPipeline.convert_image>` must now accept a
``response_body`` parameter.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
New features
~~~~~~~~~~~~
- Applied black_ coding style to files generated with the
:command:`genspider` and :command:`startproject` commands.
(:issue:`5809`, :issue:`5814`)
.. _black: https://black.readthedocs.io/en/stable/
- :setting:`FEED_EXPORT_ENCODING` is now set to ``"utf-8"`` in the
``settings.py`` file that the :command:`startproject` command generates.
With this value, JSON exports won’t force the use of escape sequences for
non-ASCII characters.
(:issue:`5797`, :issue:`5800`)
- The :class:`~scrapy.extensions.memusage.MemoryUsage` extension now logs the
peak memory usage during checks, and the binary unit MiB is now used to
avoid confusion.
(:issue:`5717`, :issue:`5722`, :issue:`5727`)
- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set
to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from
``None``, as the latter indicates that the default spider callback
(:meth:`~scrapy.Spider.parse`) is to be used.
(:issue:`5798`)
Bug fixes
~~~~~~~~~
- Enabled unsafe legacy SSL renegotiation to fix access to some outdated
websites.
(:issue:`5491`, :issue:`5790`)
- Fixed STARTTLS-based email delivery not working with Twisted 21.2.0 and
better.
(:issue:`5386`, :issue:`5406`)
- Fixed the :meth:`finish_exporting` method of :ref:`item exporters
<topics-exporters>` not being called for empty files.
(:issue:`5537`, :issue:`5758`)
- Fixed HTTP/2 responses getting only the last value for a header when
multiple headers with the same name are received.
(:issue:`5777`)
- Fixed an exception raised by the :command:`shell` command on some cases
when :ref:`using asyncio <using-asyncio>`.
(:issue:`5740`, :issue:`5742`, :issue:`5748`, :issue:`5759`, :issue:`5760`,
:issue:`5771`)
- When using :class:`~scrapy.spiders.CrawlSpider`, callback keyword arguments
(``cb_kwargs``) added to a request in the ``process_request`` callback of a
:class:`~scrapy.spiders.Rule` will no longer be ignored.
(:issue:`5699`)
- The :ref:`images pipeline <images-pipeline>` no longer re-encodes JPEG
files.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
- Fixed the handling of transparent WebP images by the :ref:`images pipeline
<images-pipeline>`.
(:issue:`3072`, :issue:`5766`, :issue:`5767`)
- :func:`scrapy.shell.inspect_response` no longer inhibits ``SIGINT``
(Ctrl+C).
(:issue:`2918`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
with ``unique=False`` no longer filters out links that have identical URL
*and* text.
(:issue:`3798`, :issue:`3799`, :issue:`4695`, :issue:`5458`)
- :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` now
ignores URL protocols that do not support ``robots.txt`` (``data://``,
``file://``).
(:issue:`5807`)
- Silenced the ``filelock`` debug log messages introduced in Scrapy 2.6.
(:issue:`5753`, :issue:`5754`)
- Fixed the output of ``scrapy -h`` showing an unintended ``**commands**``
line.
(:issue:`5709`, :issue:`5711`, :issue:`5712`)
- Made the active project indication in the output of :ref:`commands
<topics-commands>` more clear.
(:issue:`5715`)
Documentation
~~~~~~~~~~~~~
- Documented how to :ref:`debug spiders from Visual Studio Code
<debug-vscode>`.
(:issue:`5721`)
- Documented how :setting:`DOWNLOAD_DELAY` affects per-domain concurrency.
(:issue:`5083`, :issue:`5540`)
- Improved consistency.
(:issue:`5761`)
- Fixed typos.
(:issue:`5714`, :issue:`5744`, :issue:`5764`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Applied :ref:`black coding style <coding-style>`, sorted import statements,
and introduced :ref:`pre-commit <scrapy-pre-commit>`.
(:issue:`4654`, :issue:`4658`, :issue:`5734`, :issue:`5737`, :issue:`5806`,
:issue:`5810`)
- Switched from :mod:`os.path` to :mod:`pathlib`.
(:issue:`4916`, :issue:`4497`, :issue:`5682`)
- Addressed many issues reported by Pylint.
(:issue:`5677`)
- Improved code readability.
(:issue:`5736`)
- Improved package metadata.
(:issue:`5768`)
- Removed direct invocations of ``setup.py``.
(:issue:`5774`, :issue:`5776`)
- Removed unnecessary :class:`~collections.OrderedDict` usages.
(:issue:`5795`)
- Removed unnecessary ``__str__`` definitions.
(:issue:`5150`)
- Removed obsolete code and comments.
(:issue:`5725`, :issue:`5729`, :issue:`5730`, :issue:`5732`)
- Fixed test and CI issues.
(:issue:`5749`, :issue:`5750`, :issue:`5756`, :issue:`5762`, :issue:`5765`,
:issue:`5780`, :issue:`5781`, :issue:`5782`, :issue:`5783`, :issue:`5785`,
:issue:`5786`)
.. _release-2.7.1:
Scrapy 2.7.1 (2022-11-02)
@ -12,7 +218,7 @@ New features
~~~~~~~~~~~~
- Relaxed the restriction introduced in 2.6.2 so that the
``Proxy-Authentication`` header can again be set explicitly, as long as the
``Proxy-Authorization`` header can again be set explicitly, as long as the
proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and
for as long as that proxy URL remains the same; this restores compatibility
with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`).
@ -281,7 +487,7 @@ Scrapy 2.6.2 (2022-07-25)
processes a request with :reqmeta:`proxy` metadata, and that
:reqmeta:`proxy` metadata includes proxy credentials,
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
the ``Proxy-Authentication`` header, but only if that header is not already
the ``Proxy-Authorization`` header, but only if that header is not already
set.
There are third-party proxy-rotation downloader middlewares that set
@ -294,7 +500,7 @@ Scrapy 2.6.2 (2022-07-25)
These third-party proxy-rotation downloader middlewares could change the
:reqmeta:`proxy` metadata of a request to a new value, but fail to remove
the ``Proxy-Authentication`` header from the previous value of the
the ``Proxy-Authorization`` header from the previous value of the
:reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
to a different proxy.
@ -2250,7 +2456,7 @@ Scrapy 1.8.3 (2022-07-25)
processes a request with :reqmeta:`proxy` metadata, and that
:reqmeta:`proxy` metadata includes proxy credentials,
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
the ``Proxy-Authentication`` header, but only if that header is not already
the ``Proxy-Authorization`` header, but only if that header is not already
set.
There are third-party proxy-rotation downloader middlewares that set
@ -2263,7 +2469,7 @@ Scrapy 1.8.3 (2022-07-25)
These third-party proxy-rotation downloader middlewares could change the
:reqmeta:`proxy` metadata of a request to a new value, but fail to remove
the ``Proxy-Authentication`` header from the previous value of the
the ``Proxy-Authorization`` header from the previous value of the
:reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
to a different proxy.
@ -4700,7 +4906,7 @@ Scrapy 0.22.1 (released 2014-02-08)
- BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
- BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
- Fix tests for Travis-CI build (:commit:`76c7e20`)
- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
- Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
- Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
@ -4725,7 +4931,7 @@ Enhancements
- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
- Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`)
- Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
- Selectors register EXSLT namespaces by default (:issue:`472`)
- Unify item loaders similar to selectors renaming (:issue:`461`)
@ -4905,7 +5111,7 @@ Scrapy 0.18.0 (released 2013-08-09)
-----------------------------------
- Lot of improvements to testsuite run using Tox, including a way to test on pypi
- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`)
- Use lxml recover option to parse sitemaps (:issue:`347`)
- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
@ -4939,8 +5145,8 @@ Scrapy 0.18.0 (released 2013-08-09)
- Added ``--pdb`` option to ``scrapy`` command line tool
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
- Several improvements to spider contracts
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections,
- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62
- added from_crawler method to spiders
- added system tests with mock server
- more improvements to macOS compatibility (thanks Alex Cepoi)
@ -5082,7 +5288,7 @@ Scrapy changes:
- promoted :ref:`topics-djangoitem` to main contrib
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- removed signal: ``scrapy.mail.mail_sent``
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
- DBM is now the default storage backend for HTTP cache middleware
@ -5148,7 +5354,7 @@ Scrapy 0.14
New features and settings
~~~~~~~~~~~~~~~~~~~~~~~~~
- Support for `AJAX crawleable urls`_
- Support for `AJAX crawlable urls`_
- New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
- added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
- Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
@ -5408,7 +5614,7 @@ Backward-incompatible changes
- Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
- Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
- Refactored HTTP Cache middleware
- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
- Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
- Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
- Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
@ -5419,7 +5625,7 @@ Scrapy 0.7
First release of Scrapy.
.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/

2
docs/topics/broad-crawls.rst
View File

@ -68,7 +68,7 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
to increase it will depend on how much CPU and memory you crawler will have
to increase it will depend on how much CPU and memory your crawler will have
available.
A good starting point is ``100``::

32
docs/topics/debug.rst
View File

@ -19,7 +19,7 @@ Consider the following Scrapy spider below::
def parse(self, response):
# <processing code not shown>
# collect `item_urls`
# collect `item_urls`
for item_url in item_urls:
yield scrapy.Request(item_url, self.parse_item)
@ -150,3 +150,33 @@ available in all future runs should they be necessary again::
For more information, check the :ref:`topics-logging` section.
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
.. _debug-vscode:
Visual Studio Code
==================
.. highlight:: json
To debug spiders with Visual Studio Code you can use the following ``launch.json``::
{
"version": "0.1.0",
"configurations": [
{
"name": "Python: Launch Scrapy Spider",
"type": "python",
"request": "launch",
"module": "scrapy",
"args": [
"runspider",
"${file}"
],
"console": "integratedTerminal"
}
]
}
Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in
your Scrapy spider.

4
docs/topics/feed-exports.rst
View File

@ -515,6 +515,10 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
Use ``utf-8`` if you want UTF-8 for JSON too.
.. versionchanged:: 2.8
The :command:`startproject` command now sets this setting to
``utf-8`` in the generated ``settings.py`` file.
.. setting:: FEED_EXPORT_FIELDS
FEED_EXPORT_FIELDS

11
docs/topics/item-pipeline.rst
View File

@ -186,10 +186,12 @@ item.
::
import hashlib
from pathlib import Path
from urllib.parse import quote
import scrapy
from itemadapter import ItemAdapter
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.defer import maybe_deferred_to_future
@ -203,8 +205,10 @@ item.
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await maybe_deferred_to_future(
spider.crawler.engine.download(request, spider)
)
if response.status != 200:
# Error happened, return item.
@ -214,8 +218,7 @@ item.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
# Store filename in item.
adapter["screenshot_filename"] = filename

9
docs/topics/media-pipeline.rst
View File

@ -156,7 +156,6 @@ By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.

43
docs/topics/request-response.rst
View File

@ -32,11 +32,20 @@ Request objects
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
request (once it's downloaded) as its first parameter.
In addition to a function, the following values are supported:
- ``None`` (default), which indicates that the spider's
:meth:`~scrapy.Spider.parse` method must be used.
- :func:`~scrapy.http.request.NO_CALLBACK`
For more information, see
:ref:`topics-request-response-ref-request-callback-arguments`.
.. note:: If exceptions are raised during processing, ``errback`` is
called instead.
:type callback: collections.abc.Callable
@ -69,16 +78,24 @@ Request objects
1. Using a dict::
request_with_cookies = Request(url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'})
request_with_cookies = Request(
url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'},
)
2. Using a list of dicts::
request_with_cookies = Request(url="http://www.example.com",
cookies=[{'name': 'currency',
'value': 'USD',
'domain': 'example.com',
'path': '/currency'}])
request_with_cookies = Request(
url="http://www.example.com",
cookies=[
{
'name': 'currency',
'value': 'USD',
'domain': 'example.com',
'path': '/currency',
},
],
)
The latter form allows for customizing the ``domain`` and ``path``
attributes of the cookie. This is only useful if the cookies are saved
@ -228,6 +245,8 @@ Request objects
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.http.request.NO_CALLBACK
.. autofunction:: scrapy.utils.request.request_from_dict

37
docs/topics/settings.rst
View File

@ -636,24 +636,41 @@ DOWNLOAD_DELAY
Default: ``0``
The amount of time (in secs) that the downloader should wait before downloading
consecutive pages from the same website. This can be used to throttle the
crawling speed to avoid hitting servers too hard. Decimal numbers are
supported. Example::
Minimum seconds to wait between 2 consecutive requests to the same domain.
DOWNLOAD_DELAY = 0.25 # 250 ms of delay
Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
servers too hard.
Decimal numbers are supported. For example, to send a maximum of 4 requests
every 10 seconds::
DOWNLOAD_DELAY = 2.5
This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
setting (which is enabled by default). By default, Scrapy doesn't wait a fixed
amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`.
setting, which is enabled by default.
When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
per ip address instead of per domain.
per IP address instead of per domain.
Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
concurrency for that domain is 1. When testing throttling configurations, it
usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first,
and only increase :setting:`DOWNLOAD_DELAY` once
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is
desired.
.. _spider-download_delay-attribute:
You can also change this setting per spider by setting ``download_delay``
spider attribute.
.. note::
This delay can be set per spider using :attr:`download_delay` spider attribute.
It is also possible to change this setting per domain, although it requires
non-trivial code. See the implementation of the :ref:`AutoThrottle
<topics-autothrottle>` extension for an example.
.. setting:: DOWNLOAD_HANDLERS

2
docs/topics/spiders.rst
View File

@ -99,7 +99,7 @@ scrapy.Spider
.. attribute:: crawler
This attribute is set by the :meth:`from_crawler` class method after
initializating the class, and links to the
initializing the class, and links to the
:class:`~scrapy.crawler.Crawler` object to which this spider instance is
bound.

16
docs/utils/linkfix.py Executable file → Normal file
View File

@ -13,6 +13,8 @@ Author: dufferzafar
"""
import re
import sys
from pathlib import Path
def main():
@ -23,15 +25,15 @@ def main():
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(r"(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))")
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
sys.exit(1)
# For every line, fix the respective file
for line in output_lines:
@ -51,14 +53,12 @@ def main():
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
Path(_filename).write_text(_contents, encoding="utf-8")
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = Path(_filename).read_text(encoding="utf-8")
_contents = _contents.replace(match.group(3), match.group(4))
else:
@ -66,5 +66,5 @@ def main():
print("Not Understood: " + line)
if __name__ == '__main__':
if __name__ == "__main__":
main()

22
extras/qps-bench-server.py
View File

@ -1,13 +1,13 @@
#!/usr/bin/env python
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from time import time
from twisted.internet import reactor
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
class Root(Resource):
def __init__(self):
Resource.__init__(self)
self.concurrent = 0
@ -26,9 +26,9 @@ class Root(Resource):
delta = now - self.lasttime
# reset stats on high iter-request times caused by client restarts
if delta > 3: # seconds
if delta > 3: # seconds
self._reset_stats()
return ''
return ""
self.tail.appendleft(delta)
self.lasttime = now
@ -37,15 +37,17 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
print(
f"samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}"
)
if 'latency' in request.args:
latency = float(request.args['latency'][0])
if "latency" in request.args:
latency = float(request.args["latency"][0])
reactor.callLater(latency, self._finish, request)
return NOT_DONE_YET
self.concurrent -= 1
return ''
return ""
def _finish(self, request):
self.concurrent -= 1

17
extras/qpsclient.py
View File

@ -1,25 +1,26 @@
"""
A spider that generate light requests to meassure QPS throughput
A spider that generate light requests to measure QPS throughput
usage:
scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0
--set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
"""
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.spiders import Spider
class QPSSpider(Spider):
name = 'qps'
benchurl = 'http://localhost:8880/'
name = "qps"
benchurl = "http://localhost:8880/"
# Max concurrency is limited by global CONCURRENT_REQUESTS setting
max_concurrent_requests = 8
# Requests per second goal
qps = None # same as: 1 / download_delay
qps = None # same as: 1 / download_delay
download_delay = None
# time in seconds to delay server responses
latency = None
@ -37,11 +38,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += f'?latency={self.latency}'
url += f"?latency={self.latency}"
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
urls = [url.replace("localhost", f"127.0.0.{x + 1}") for x in range(slots)]
else:
urls = [url]

28
pylintrc
View File

@ -9,33 +9,26 @@ disable=abstract-method,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bare-except,
blacklisted-name,
broad-except,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-dict-items,
consider-using-from-import,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
disallowed-name,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
fixme,
function-redefined,
global-statement,
implicit-str-concat,
import-error,
import-outside-toplevel,
import-self,
@ -52,18 +45,12 @@ disable=abstract-method,
lost-exception,
method-hidden,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-statement,
pointless-string-statement,
@ -74,10 +61,7 @@ disable=abstract-method,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
@ -89,31 +73,23 @@ disable=abstract-method,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-dunder-call,
unnecessary-lambda,
unnecessary-pass,
unreachable,
unspecified-encoding,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

2
scrapy/VERSION
View File

@ -1 +1 @@
2.7.1
2.8.0

22
scrapy/__init__.py
View File

@ -9,21 +9,27 @@ import warnings
from twisted import version as _txv
# Declare top-level shortcuts
from scrapy.spiders import Spider
from scrapy.http import Request, FormRequest
from scrapy.http import FormRequest, Request
from scrapy.item import Field, Item
from scrapy.selector import Selector
from scrapy.item import Item, Field
from scrapy.spiders import Spider
__all__ = [
'__version__', 'version_info', 'twisted_version', 'Spider',
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
"__version__",
"version_info",
"twisted_version",
"Spider",
"Request",
"FormRequest",
"Selector",
"Item",
"Field",
]
# Scrapy and Twisted versions
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
@ -34,7 +40,7 @@ if sys.version_info < (3, 7):
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
del pkgutil

2
scrapy/__main__.py
View File

@ -1,4 +1,4 @@
from scrapy.cmdline import execute
if __name__ == '__main__':
if __name__ == "__main__":
execute()

48
scrapy/cmdline.py
View File

@ -1,30 +1,31 @@
import sys
import os
import argparse
import cProfile
import inspect
import os
import sys
import pkg_resources
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == '-:':
if arg_string[:2] == "-:":
return None
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# TODO: add `name` attribute to commands and merge this function with
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
for obj in vars(module).values():
@ -32,7 +33,7 @@ def _iter_command_classes(module_name):
inspect.isclass(obj)
and issubclass(obj, ScrapyCommand)
and obj.__module__ == module.__name__
and not obj == ScrapyCommand
and obj not in (ScrapyCommand, BaseRunSpiderCommand)
):
yield obj
@ -41,12 +42,12 @@ def _get_commands_from_module(module, inproject):
d = {}
for cmd in _iter_command_classes(module):
if inproject or not cmd.requires_project:
cmdname = cmd.__module__.split('.')[-1]
cmdname = cmd.__module__.split(".")[-1]
d[cmdname] = cmd()
return d
def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
cmds = {}
for entry_point in pkg_resources.iter_entry_points(group):
obj = entry_point.load()
@ -58,9 +59,9 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
def _get_commands_dict(settings, inproject):
cmds = _get_commands_from_module('scrapy.commands', inproject)
cmds = _get_commands_from_module("scrapy.commands", inproject)
cmds.update(_get_commands_from_entry_points(inproject))
cmds_module = settings['COMMANDS_MODULE']
cmds_module = settings["COMMANDS_MODULE"]
if cmds_module:
cmds.update(_get_commands_from_module(cmds_module, inproject))
return cmds
@ -69,7 +70,7 @@ def _get_commands_dict(settings, inproject):
def _pop_command_name(argv):
i = 0
for arg in argv[1:]:
if not arg.startswith('-'):
if not arg.startswith("-"):
del argv[i]
return arg
i += 1
@ -78,7 +79,8 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
else:
print(f"Scrapy {version} - no active project\n")
@ -123,11 +125,11 @@ def execute(argv=None, settings=None):
settings = get_project_settings()
# set EDITOR from environment if available
try:
editor = os.environ['EDITOR']
editor = os.environ["EDITOR"]
except KeyError:
pass
else:
settings['EDITOR'] = editor
settings["EDITOR"] = editor
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
@ -140,11 +142,13 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
parser = ScrapyArgumentParser(
formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler="resolve",
description=cmd.long_desc(),
)
settings.setdict(cmd.default_settings, priority="command")
cmd.settings = settings
cmd.add_options(parser)
opts, args = parser.parse_known_args(args=argv[1:])
@ -167,12 +171,12 @@ def _run_command_profiled(cmd, args, opts):
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)
p.runctx("cmd.run(args, opts)", globals(), loc)
if opts.profile:
p.dump_stats(opts.profile)
if __name__ == '__main__':
if __name__ == "__main__":
try:
execute()
finally:

138
scrapy/commands/__init__.py
View File

@ -1,31 +1,33 @@
"""
Base class for Scrapy commands
"""
import os
import argparse
from typing import Any, Dict
import os
from pathlib import Path
from typing import Any, Dict, Optional
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
class ScrapyCommand:
requires_project = False
crawler_process = None
crawler_process: Optional[CrawlerProcess] = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}
exitcode = 0
def __init__(self):
self.settings = None # set in scrapy.cmdline
def __init__(self) -> None:
self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
if hasattr(self, '_crawler'):
if hasattr(self, "_crawler"):
raise RuntimeError("crawler already set")
self._crawler = crawler
@ -59,42 +61,58 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
group = parser.add_argument_group(title='Global Options')
group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_argument("--nolog", action="store_true",
help="disable logging completely")
group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group = parser.add_argument_group(title="Global Options")
group.add_argument(
"--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
)
group.add_argument(
"-L",
"--loglevel",
metavar="LEVEL",
default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})",
)
group.add_argument(
"--nolog", action="store_true", help="disable logging completely"
)
group.add_argument(
"--profile",
metavar="FILE",
default=None,
help="write python cProfile stats to FILE",
)
group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE")
group.add_argument(
"-s",
"--set",
action="append",
default=[],
metavar="NAME=VALUE",
help="set/override setting (may be repeated)",
)
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
self.settings.setdict(arglist_to_dict(opts.set),
priority='cmdline')
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
if opts.logfile:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
self.settings.set("LOG_ENABLED", True, priority="cmdline")
self.settings.set("LOG_FILE", opts.logfile, priority="cmdline")
if opts.loglevel:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
self.settings.set("LOG_ENABLED", True, priority="cmdline")
self.settings.set("LOG_LEVEL", opts.loglevel, priority="cmdline")
if opts.nolog:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
self.settings.set("LOG_ENABLED", False, priority="cmdline")
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
Path(opts.pidfile).write_text(
str(os.getpid()) + os.linesep, encoding="utf-8"
)
if opts.pdb:
failure.startDebugMode()
@ -110,18 +128,39 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
parser.add_argument(
"-a",
dest="spargs",
action="append",
default=[],
metavar="NAME=VALUE",
help="set spider argument (may be repeated)",
)
parser.add_argument(
"-o",
"--output",
metavar="FILE",
action="append",
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)",
)
parser.add_argument(
"-O",
"--overwrite-output",
metavar="FILE",
action="append",
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
)
parser.add_argument(
"-t",
"--output-format",
metavar="FORMAT",
help="format to use for dumping items",
)
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -136,16 +175,21 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
self.settings.set("FEEDS", feeds, priority="cmdline")
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
super().__init__(
prog,
indent_increment=indent_increment,
max_help_position=max_help_position,
width=width,
)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
@ -156,11 +200,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
part_strings[0] = "Usage\n=====\n " + part_strings[0][len("usage: ") :]
headings = [
i for i in range(len(part_strings)) if part_strings[i].endswith(":\n")
]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
char = "-" if "Global Options" in part_strings[index] else "="
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
underline = "".join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

24
scrapy/commands/bench.py
View File

@ -1,6 +1,6 @@
import subprocess
import sys
import time
import subprocess
from urllib.parse import urlencode
import scrapy
@ -11,9 +11,9 @@ from scrapy.linkextractors import LinkExtractor
class Command(ScrapyCommand):
default_settings = {
'LOG_LEVEL': 'INFO',
'LOGSTATS_INTERVAL': 1,
'CLOSESPIDER_TIMEOUT': 10,
"LOG_LEVEL": "INFO",
"LOGSTATS_INTERVAL": 1,
"CLOSESPIDER_TIMEOUT": 10,
}
def short_desc(self):
@ -26,12 +26,11 @@ class Command(ScrapyCommand):
class _BenchServer:
def __enter__(self):
from scrapy.utils.test import get_testenv
pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver']
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE,
env=get_testenv())
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
self.proc.stdout.readline()
def __exit__(self, exc_type, exc_value, traceback):
@ -42,15 +41,16 @@ class _BenchServer:
class _BenchSpider(scrapy.Spider):
"""A spider that follows all links"""
name = 'follow'
name = "follow"
total = 10000
show = 20
baseurl = 'http://localhost:8998'
baseurl = "http://localhost:8998"
link_extractor = LinkExtractor()
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
qargs = {"total": self.total, "show": self.show}
url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

32
scrapy/commands/check.py
View File

@ -1,11 +1,12 @@
import time
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
from unittest import TextTestResult as _TextTestResult
from unittest import TextTestRunner
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object, set_environ
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):
@ -39,7 +40,7 @@ class TextTestResult(_TextTestResult):
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "[options] <spider>"
@ -49,14 +50,25 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
parser.add_argument(
"-l",
"--list",
dest="list",
action="store_true",
help="only list contracts, without checking them",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
default=False,
action="store_true",
help="print contract tests for all spiders",
)
def run(self, args, opts):
# load contracts
contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
conman = ContractsManager(load_object(c) for c in contracts)
runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
@ -66,7 +78,7 @@ class Command(ScrapyCommand):
spider_loader = self.crawler_process.spider_loader
with set_environ(SCRAPY_CHECK='true'):
with set_environ(SCRAPY_CHECK="true"):
for spidername in args or spider_loader.list():
spidercls = spider_loader.load(spidername)
spidercls.start_requests = lambda s: conman.from_spider(s, result)
@ -85,7 +97,7 @@ class Command(ScrapyCommand):
continue
print(spider)
for method in sorted(methods):
print(f' * {method}')
print(f" * {method}")
else:
start = time.time()
self.crawler_process.start()

11
scrapy/commands/crawl.py
View File

@ -16,18 +16,23 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
raise UsageError(
"running 'scrapy crawl' with more than one spider is not supported"
)
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
if getattr(crawl_defer, "result", None) is not None and issubclass(
crawl_defer.result.type, Exception
):
self.exitcode = 1
else:
self.crawler_process.start()
if (
self.crawler_process.bootstrap_failed
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
or hasattr(self.crawler_process, "has_exception")
and self.crawler_process.has_exception
):
self.exitcode = 1

14
scrapy/commands/edit.py
View File

@ -1,5 +1,5 @@
import sys
import os
import sys
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
@ -8,7 +8,7 @@ from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "<spider>"
@ -17,8 +17,10 @@ class Command(ScrapyCommand):
return "Edit spider"
def long_desc(self):
return ("Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting")
return (
"Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting"
)
def _err(self, msg):
sys.stderr.write(msg + os.linesep)
@ -28,12 +30,12 @@ class Command(ScrapyCommand):
if len(args) != 1:
raise UsageError()
editor = self.settings['EDITOR']
editor = self.settings["EDITOR"]
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
sfile = sfile.replace(".pyc", ".py")
self.exitcode = os.system(f'{editor} "{sfile}"')

44
scrapy/commands/fetch.py
View File

@ -1,11 +1,12 @@
import sys
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
class Command(ScrapyCommand):
@ -27,38 +28,51 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument(
"--headers",
dest="headers",
action="store_true",
help="print response HTTP headers instead of body",
)
parser.add_argument(
"--no-redirect",
dest="no_redirect",
action="store_true",
default=False,
help="do not handle HTTP 3xx status codes and print response as-is",
)
def _print_headers(self, headers, prefix):
for key, values in headers.items():
for value in values:
self._print_bytes(prefix + b' ' + key + b': ' + value)
self._print_bytes(prefix + b" " + key + b": " + value)
def _print_response(self, response, opts):
if opts.headers:
self._print_headers(response.request.headers, b'>')
print('>')
self._print_headers(response.headers, b'<')
self._print_headers(response.request.headers, b">")
print(">")
self._print_headers(response.headers, b"<")
else:
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
sys.stdout.buffer.write(bytes_ + b'\n')
sys.stdout.buffer.write(bytes_ + b"\n")
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
request = Request(
args[0],
callback=self._print_response,
cb_kwargs={"opts": opts},
dont_filter=True,
)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:
request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
request.meta["handle_httpstatus_list"] = SequenceExclude(range(300, 400))
else:
request.meta['handle_httpstatus_all'] = True
request.meta["handle_httpstatus_all"] = True
spidercls = DefaultSpider
spider_loader = self.crawler_process.spider_loader

138
scrapy/commands/genspider.py
View File

@ -1,15 +1,15 @@
import os
import shutil
import string
from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext
from pathlib import Path
from typing import Optional, cast
from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
def sanitize_module_name(module_name):
@ -17,7 +17,7 @@ def sanitize_module_name(module_name):
with underscores and prefixing it with a letter if it doesn't start
with one
"""
module_name = module_name.replace('-', '_').replace('.', '_')
module_name = module_name.replace("-", "_").replace(".", "_")
if module_name[0] not in string.ascii_letters:
module_name = "a" + module_name
return module_name
@ -26,7 +26,7 @@ def sanitize_module_name(module_name):
def extract_domain(url):
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == '' and o.netloc == '':
if o.scheme == "" and o.netloc == "":
o = urlparse("//" + url.lstrip("/"))
return o.netloc
@ -34,7 +34,7 @@ def extract_domain(url):
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "[options] <name> <domain>"
@ -44,16 +44,40 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
parser.add_argument(
"-l",
"--list",
dest="list",
action="store_true",
help="List available templates",
)
parser.add_argument(
"-e",
"--edit",
dest="edit",
action="store_true",
help="Edit spider after creating it",
)
parser.add_argument(
"-d",
"--dump",
dest="dump",
metavar="TEMPLATE",
help="Dump template to standard output",
)
parser.add_argument(
"-t",
"--template",
dest="template",
default="basic",
help="Uses a custom template.",
)
parser.add_argument(
"--force",
dest="force",
action="store_true",
help="If the spider already exists, overwrite it with the template",
)
def run(self, args, opts):
if opts.list:
@ -62,8 +86,7 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
with open(template_file, "r") as f:
print(f.read())
print(template_file.read_text(encoding="utf-8"))
return
if len(args) != 2:
raise UsageError()
@ -72,7 +95,7 @@ class Command(ScrapyCommand):
domain = extract_domain(url)
module = sanitize_module_name(name)
if self.settings.get('BOT_NAME') == module:
if self.settings.get("BOT_NAME") == module:
print("Cannot create a spider with the same name as your project")
return
@ -87,50 +110,58 @@ class Command(ScrapyCommand):
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': f'{capitalized_module}Spider'
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
"domain": domain,
"classname": f"{capitalized_module}Spider",
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = abspath(dirname(spiders_module.__file__))
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
spiders_dir = "."
spider_file = f"{join(spiders_dir, module)}.py"
spiders_dir = Path(".")
spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
print(
f"Created spider {name!r} using template {template_name!r} ",
end=("" if spiders_module else "\n"),
)
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
def _find_template(self, template: str) -> Optional[Path]:
template_file = Path(self.templates_dir, f"{template}.tmpl")
if template_file.exists():
return template_file
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
return None
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(f" {splitext(filename)[0]}")
for file in sorted(Path(self.templates_dir).iterdir()):
if file.suffix == ".tmpl":
print(f" {file.stem}")
def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'):
def _spider_exists(self, name: str) -> bool:
if not self.settings.get("NEWSPIDER_MODULE"):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
path = Path(name + ".py")
if path.exists():
print(f"{path.resolve()} already exists")
return True
return False
assert (
self.crawler_process is not None
), "crawler_process must be set before calling run"
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
@ -142,18 +173,21 @@ class Command(ScrapyCommand):
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(cast(str, spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():
print(f"{path} already exists")
return True
return False
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'spiders'
def templates_dir(self) -> str:
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
"spiders",
)
)

2
scrapy/commands/list.py
View File

@ -4,7 +4,7 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def short_desc(self):
return "List available spiders"

161
scrapy/commands/parse.py
View File

@ -2,17 +2,15 @@ import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import maybeDeferred
from w3lib.url import is_url
from twisted.internet.defer import maybeDeferred
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)
@ -34,28 +32,72 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %(default)s]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
parser.add_argument(
"--spider",
dest="spider",
default=None,
help="use this spider without looking for one",
)
parser.add_argument(
"--pipelines", action="store_true", help="process items through pipelines"
)
parser.add_argument(
"--nolinks",
dest="nolinks",
action="store_true",
help="don't show links to follow (extracted requests)",
)
parser.add_argument(
"--noitems",
dest="noitems",
action="store_true",
help="don't show scraped items",
)
parser.add_argument(
"--nocolour",
dest="nocolour",
action="store_true",
help="avoid using pygments to colorize the output",
)
parser.add_argument(
"-r",
"--rules",
dest="rules",
action="store_true",
help="use CrawlSpider rules to discover the callback",
)
parser.add_argument(
"-c",
"--callback",
dest="callback",
help="use this callback for parsing, instead looking for a callback",
)
parser.add_argument(
"-m",
"--meta",
dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string",
)
parser.add_argument(
"--cbkwargs",
dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string",
)
parser.add_argument(
"-d",
"--depth",
dest="depth",
type=int,
default=1,
help="maximum depth for parsing requests [default: %(default)s]",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
action="store_true",
help="print each depth level one by one",
)
@property
def max_level(self):
@ -100,13 +142,13 @@ class Command(BaseRunSpiderCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print(f'\n>>> DEPTH LEVEL: {level} <<<')
print(f"\n>>> DEPTH LEVEL: {level} <<<")
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
print(f"\n>>> STATUS DEPTH LEVEL {self.max_level} <<<")
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:
@ -127,14 +169,16 @@ class Command(BaseRunSpiderCommand):
return d
def get_callback_from_rules(self, spider, response):
if getattr(spider, 'rules', None):
if getattr(spider, "rules", None):
for rule in spider.rules:
if rule.link_extractor.matches(response.url):
return rule.callback or "parse"
else:
logger.error('No CrawlSpider rules found in spider %(spider)r, '
'please specify a callback to use for parsing',
{'spider': spider.name})
logger.error(
"No CrawlSpider rules found in spider %(spider)r, "
"please specify a callback to use for parsing",
{"spider": spider.name},
)
def set_spidercls(self, url, opts):
spider_loader = self.crawler_process.spider_loader
@ -142,15 +186,17 @@ class Command(BaseRunSpiderCommand):
try:
self.spidercls = spider_loader.load(opts.spider)
except KeyError:
logger.error('Unable to find spider: %(spider)s',
{'spider': opts.spider})
logger.error(
"Unable to find spider: %(spider)s", {"spider": opts.spider}
)
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
logger.error("Unable to find spider for: %(url)s", {"url": url})
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
if self.spidercls:
self.spidercls.start_requests = _start_requests
@ -160,8 +206,7 @@ class Command(BaseRunSpiderCommand):
self.crawler_process.start()
if not self.first_response:
logger.error('No response downloaded for: %(url)s',
{'url': url})
logger.error("No response downloaded for: %(url)s", {"url": url})
def scraped_data(self, args):
items, requests, opts, depth, spider, callback = args
@ -175,8 +220,8 @@ class Command(BaseRunSpiderCommand):
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.meta["_depth"] = depth + 1
req.meta["_callback"] = req.callback
req.callback = callback
scraped_data += requests
@ -189,7 +234,7 @@ class Command(BaseRunSpiderCommand):
self.first_response = response
# determine real callback
cb = response.meta['_callback']
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
@ -197,23 +242,27 @@ class Command(BaseRunSpiderCommand):
cb = self.get_callback_from_rules(spider, response)
if not cb:
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
{'url': response.url, 'spider': spider.name})
logger.error(
"Cannot find a rule that matches %(url)r in spider: %(spider)s",
{"url": response.url, "spider": spider.name},
)
return
else:
cb = 'parse'
cb = "parse"
if not callable(cb):
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
{'callback': cb, 'spider': spider.name})
logger.error(
"Cannot find callback %(callback)r in spider: %(spider)s",
{"callback": cb, "spider": spider.name},
)
return
# parse items and requests
depth = response.meta['_depth']
depth = response.meta["_depth"]
d = self.run_callback(response, cb, cb_kwargs)
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
@ -228,8 +277,8 @@ class Command(BaseRunSpiderCommand):
if opts.cbkwargs:
request.cb_kwargs.update(opts.cbkwargs)
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.meta["_depth"] = 1
request.meta["_callback"] = request.callback
request.callback = callback
return request
@ -244,16 +293,22 @@ class Command(BaseRunSpiderCommand):
try:
opts.meta = json.loads(opts.meta)
except ValueError:
raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
"Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
raise UsageError(
"Invalid -m/--meta value, pass a valid json string to -m or --meta. "
'Example: --meta=\'{"foo" : "bar"}\'',
print_help=False,
)
def process_request_cb_kwargs(self, opts):
if opts.cbkwargs:
try:
opts.cbkwargs = json.loads(opts.cbkwargs)
except ValueError:
raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
"Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
raise UsageError(
"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
'Example: --cbkwargs=\'{"foo" : "bar"}\'',
print_help=False,
)
def run(self, args, opts):
# parse arguments

34
scrapy/commands/runspider.py
View File

@ -1,32 +1,32 @@
import sys
import os
from importlib import import_module
from os import PathLike
from pathlib import Path
from types import ModuleType
from typing import Union
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext not in ('.py', '.pyw'):
def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in (".py", ".pyw"):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
dirname = str(abspath.parent)
sys.path = [dirname] + sys.path
try:
module = import_module(fname)
module = import_module(abspath.stem)
finally:
if dirname:
sys.path.pop(0)
sys.path.pop(0)
return module
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[options] <spider_file>"
@ -40,13 +40,13 @@ class Command(BaseRunSpiderCommand):
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
filename = Path(args[0])
if not filename.exists():
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError(f"Unable to load {filename!r}: {e}\n")
raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n")

40
scrapy/commands/settings.py
View File

@ -7,8 +7,7 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[options]"
@ -18,16 +17,33 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
parser.add_argument(
"--get", dest="get", metavar="SETTING", help="print raw setting value"
)
parser.add_argument(
"--getbool",
dest="getbool",
metavar="SETTING",
help="print setting value, interpreted as a boolean",
)
parser.add_argument(
"--getint",
dest="getint",
metavar="SETTING",
help="print setting value, interpreted as an integer",
)
parser.add_argument(
"--getfloat",
dest="getfloat",
metavar="SETTING",
help="print setting value, interpreted as a float",
)
parser.add_argument(
"--getlist",
dest="getlist",
metavar="SETTING",
help="print setting value, interpreted as a list",
)
def run(self, args, opts):
settings = self.crawler_process.settings

44
scrapy/commands/shell.py
View File

@ -8,7 +8,7 @@ from threading import Thread
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.shell import Shell
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
@ -16,9 +16,9 @@ class Command(ScrapyCommand):
requires_project = False
default_settings = {
'KEEP_ALIVE': True,
'LOGSTATS_INTERVAL': 0,
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
"KEEP_ALIVE": True,
"LOGSTATS_INTERVAL": 0,
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
}
def syntax(self):
@ -28,17 +28,26 @@ class Command(ScrapyCommand):
return "Interactive scraping console"
def long_desc(self):
return ("Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file.")
return (
"Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file."
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_argument("--spider", dest="spider",
help="use this spider")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument(
"-c",
dest="code",
help="evaluate the code in the shell, print the result and exit",
)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument(
"--no-redirect",
dest="no_redirect",
action="store_true",
default=False,
help="do not handle HTTP 3xx status codes and print response as-is",
)
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
@ -58,8 +67,9 @@ class Command(ScrapyCommand):
if opts.spider:
spidercls = spider_loader.load(opts.spider)
elif url:
spidercls = spidercls_for_request(spider_loader, Request(url),
spidercls, log_multiple=True)
spidercls = spidercls_for_request(
spider_loader, Request(url), spidercls, log_multiple=True
)
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
@ -74,7 +84,9 @@ class Command(ScrapyCommand):
shell.start(url=url, redirect=not opts.no_redirect)
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
t = Thread(
target=self.crawler_process.start,
kwargs={"stop_after_crawl": False, "install_signal_handlers": False},
)
t.daemon = True
t.start()

94
scrapy/commands/startproject.py
View File

@ -1,26 +1,25 @@
import re
import os
import re
import string
from importlib.util import find_spec
from os.path import join, exists, abspath
from shutil import ignore_patterns, move, copy2, copystat
from pathlib import Path
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
TEMPLATES_TO_RENDER = (
('scrapy.cfg',),
('${project_name}', 'settings.py.tmpl'),
('${project_name}', 'items.py.tmpl'),
('${project_name}', 'pipelines.py.tmpl'),
('${project_name}', 'middlewares.py.tmpl'),
("scrapy.cfg",),
("${project_name}", "settings.py.tmpl"),
("${project_name}", "items.py.tmpl"),
("${project_name}", "pipelines.py.tmpl"),
("${project_name}", "middlewares.py.tmpl"),
)
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
def _make_writable(path):
@ -31,8 +30,7 @@ def _make_writable(path):
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "<project_name> [project_dir]"
@ -45,16 +43,18 @@ class Command(ScrapyCommand):
spec = find_spec(module_name)
return spec is not None and spec.loader is not None
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
if not re.search(r"^[_a-zA-Z]\w*$", project_name):
print(
"Error: Project names must begin with a letter and contain"
" only\nletters, numbers and underscores"
)
elif _module_exists(project_name):
print(f'Error: Module {project_name!r} already exists')
print(f"Error: Module {project_name!r} already exists")
else:
return True
return False
def _copytree(self, src, dst):
def _copytree(self, src: Path, dst: Path):
"""
Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and
@ -64,19 +64,19 @@ class Command(ScrapyCommand):
https://github.com/scrapy/scrapy/pull/2005
"""
ignore = IGNORE
names = os.listdir(src)
names = [x.name for x in src.iterdir()]
ignored_names = ignore(src, names)
if not os.path.exists(dst):
os.makedirs(dst)
if not dst.exists():
dst.mkdir(parents=True)
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if os.path.isdir(srcname):
srcname = src / name
dstname = dst / name
if srcname.is_dir():
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
@ -90,36 +90,50 @@ class Command(ScrapyCommand):
raise UsageError()
project_name = args[0]
project_dir = args[0]
if len(args) == 2:
project_dir = args[1]
project_dir = Path(args[1])
else:
project_dir = Path(args[0])
if exists(join(project_dir, 'scrapy.cfg')):
if (project_dir / "scrapy.cfg").exists():
self.exitcode = 1
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}")
return
if not self._is_valid_name(project_name):
self.exitcode = 1
return
self._copytree(self.templates_dir, abspath(project_dir))
move(join(project_dir, 'module'), join(project_dir, project_name))
self._copytree(Path(self.templates_dir), project_dir.resolve())
move(project_dir / "module", project_dir / project_name)
for paths in TEMPLATES_TO_RENDER:
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
tplfile = Path(
project_dir,
*(
string.Template(s).substitute(project_name=project_name)
for s in paths
),
)
render_templatefile(
tplfile,
project_name=project_name,
ProjectName=string_camelcase(project_name),
)
print(
f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:"
)
print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:")
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'project'
def templates_dir(self) -> str:
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
"project",
)
)

12
scrapy/commands/version.py
View File

@ -5,8 +5,7 @@ from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[-v]"
@ -16,8 +15,13 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store_true",
help="also display twisted/python/platform info (useful for bug reports)",
)
def run(self, args, opts):
if opts.verbose:

8
scrapy/commands/view.py
View File

@ -1,19 +1,21 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
class Command(fetch.Command):
def short_desc(self):
return "Open URL in browser, as seen by Scrapy"
def long_desc(self):
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
return (
"Fetch a URL using the Scrapy downloader and show its contents in a browser"
)
def add_options(self, parser):
super().add_options(parser)
parser.add_argument('--headers', help=argparse.SUPPRESS)
parser.add_argument("--headers", help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)

35
scrapy/contracts/__init__.py
View File

@ -11,16 +11,17 @@ from scrapy.utils.spider import iterate_spider_output
class Contract:
""" Abstract class for contracts """
"""Abstract class for contracts"""
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
if hasattr(self, "pre_process"):
cb = request.callback
@wraps(cb)
@ -43,7 +44,7 @@ class Contract:
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
if hasattr(self, "post_process"):
cb = request.callback
@wraps(cb)
@ -88,12 +89,12 @@ class ContractsManager:
def extract_contracts(self, method):
contracts = []
for line in method.__doc__.split('\n'):
for line in method.__doc__.split("\n"):
line = line.strip()
if line.startswith('@'):
name, args = re.match(r'@(\w+)\s*(.*)', line).groups()
args = re.split(r'\s+', args)
if line.startswith("@"):
name, args = re.match(r"@(\w+)\s*(.*)", line).groups()
args = re.split(r"\s+", args)
contracts.append(self.contracts[name](method, *args))
@ -106,7 +107,7 @@ class ContractsManager:
try:
requests.append(self.from_method(bound_method, results))
except Exception:
case = _create_testcase(bound_method, 'contract')
case = _create_testcase(bound_method, "contract")
results.addError(case, sys.exc_info())
return requests
@ -124,13 +125,13 @@ class ContractsManager:
# Don't filter requests to allow
# testing different callbacks on the same URL.
kwargs['dont_filter'] = True
kwargs['callback'] = method
kwargs["dont_filter"] = True
kwargs["callback"] = method
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
args.remove('self')
args.remove("self")
# check if all positional arguments are defined in kwargs
if set(args).issubset(set(kwargs)):
@ -146,7 +147,7 @@ class ContractsManager:
return request
def _clean_req(self, request, method, results):
""" stop the request from returning objects and records any errors """
"""stop the request from returning objects and records any errors"""
cb = request.callback
@ -156,11 +157,11 @@ class ContractsManager:
output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
except Exception:
case = _create_testcase(method, 'callback')
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())
def eb_wrapper(failure):
case = _create_testcase(method, 'errback')
case = _create_testcase(method, "errback")
exc_info = failure.type, failure.value, failure.getTracebackObject()
results.addError(case, exc_info)
@ -175,6 +176,6 @@ def _create_testcase(method, desc):
def __str__(_self):
return f"[{spider}] {method.__name__} ({desc})"
name = f'{spider}_{method.__name__}'
name = f"{spider}_{method.__name__}"
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

62
scrapy/contracts/default.py
View File

@ -1,6 +1,6 @@
import json
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
@ -9,50 +9,50 @@ from scrapy.http import Request
# contracts
class UrlContract(Contract):
""" Contract to set the url of the request (mandatory)
@url http://scrapy.org
"""Contract to set the url of the request (mandatory)
@url http://scrapy.org
"""
name = 'url'
name = "url"
def adjust_request_args(self, args):
args['url'] = self.args[0]
args["url"] = self.args[0]
return args
class CallbackKeywordArgumentsContract(Contract):
""" Contract to set the keyword arguments for the request.
The value should be a JSON-encoded dictionary, e.g.:
"""Contract to set the keyword arguments for the request.
The value should be a JSON-encoded dictionary, e.g.:
@cb_kwargs {"arg1": "some value"}
@cb_kwargs {"arg1": "some value"}
"""
name = 'cb_kwargs'
name = "cb_kwargs"
def adjust_request_args(self, args):
args['cb_kwargs'] = json.loads(' '.join(self.args))
args["cb_kwargs"] = json.loads(" ".join(self.args))
return args
class ReturnsContract(Contract):
""" Contract to check the output of a callback
"""Contract to check the output of a callback
general form:
@returns request(s)/item(s) [min=1 [max]]
general form:
@returns request(s)/item(s) [min=1 [max]]
e.g.:
@returns request
@returns request 2
@returns request 2 10
@returns request 0 10
e.g.:
@returns request
@returns request 2
@returns request 2 10
@returns request 0 10
"""
name = 'returns'
name = "returns"
object_type_verifiers = {
'request': lambda x: isinstance(x, Request),
'requests': lambda x: isinstance(x, Request),
'item': is_item,
'items': is_item,
"request": lambda x: isinstance(x, Request),
"requests": lambda x: isinstance(x, Request),
"item": is_item,
"items": is_item,
}
def __init__(self, *args, **kwargs):
@ -73,7 +73,7 @@ class ReturnsContract(Contract):
try:
self.max_bound = int(self.args[2])
except IndexError:
self.max_bound = float('inf')
self.max_bound = float("inf")
def post_process(self, output):
occurrences = 0
@ -81,23 +81,25 @@ class ReturnsContract(Contract):
if self.obj_type_verifier(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
assertion = self.min_bound <= occurrences <= self.max_bound
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = f'{self.min_bound}..{self.max_bound}'
expected = f"{self.min_bound}..{self.max_bound}"
raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
raise ContractFail(
f"Returned {occurrences} {self.obj_name}, expected {expected}"
)
class ScrapesContract(Contract):
""" Contract to check presence of fields in scraped items
@scrapes page_name page_body
"""Contract to check presence of fields in scraped items
@scrapes page_name page_body
"""
name = 'scrapes'
name = "scrapes"
def post_process(self, output):
for x in output:

70
scrapy/core/downloader/__init__.py
View File

@ -1,16 +1,16 @@
import random
from time import time
from datetime import datetime
from collections import deque
from datetime import datetime
from time import time
from twisted.internet import defer, task
from scrapy import signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.resolver import dnscache
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
class Slot:
@ -41,9 +41,11 @@ class Slot:
def __repr__(self):
cls_name = self.__class__.__name__
return (f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})")
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})"
)
def __str__(self):
return (
@ -56,11 +58,11 @@ class Slot:
def _get_concurrency_delay(concurrency, spider, settings):
delay = settings.getfloat('DOWNLOAD_DELAY')
if hasattr(spider, 'download_delay'):
delay = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
delay = spider.download_delay
if hasattr(spider, 'max_concurrent_requests'):
if hasattr(spider, "max_concurrent_requests"):
concurrency = spider.max_concurrent_requests
return concurrency, delay
@ -68,7 +70,7 @@ def _get_concurrency_delay(concurrency, spider, settings):
class Downloader:
DOWNLOAD_SLOT = 'download_slot'
DOWNLOAD_SLOT = "download_slot"
def __init__(self, crawler):
self.settings = crawler.settings
@ -76,10 +78,10 @@ class Downloader:
self.slots = {}
self.active = set()
self.handlers = DownloadHandlers(crawler)
self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
self.total_concurrency = self.settings.getint("CONCURRENT_REQUESTS")
self.domain_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
self.ip_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
self.randomize_delay = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
self._slot_gc_loop = task.LoopingCall(self._slot_gc)
self._slot_gc_loop.start(60)
@ -99,7 +101,9 @@ class Downloader:
def _get_slot(self, request, spider):
key = self._get_slot_key(request, spider)
if key not in self.slots:
conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
conc = (
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
)
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
self.slots[key] = Slot(conc, delay, self.randomize_delay)
@ -109,7 +113,7 @@ class Downloader:
if self.DOWNLOAD_SLOT in request.meta:
return request.meta[self.DOWNLOAD_SLOT]
key = urlparse_cached(request).hostname or ''
key = urlparse_cached(request).hostname or ""
if self.ip_concurrency:
key = dnscache.get(key, key)
@ -124,9 +128,9 @@ class Downloader:
return response
slot.active.add(request)
self.signals.send_catch_log(signal=signals.request_reached_downloader,
request=request,
spider=spider)
self.signals.send_catch_log(
signal=signals.request_reached_downloader, request=request, spider=spider
)
deferred = defer.Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred))
self._process_queue(spider, slot)
@ -134,6 +138,7 @@ class Downloader:
def _process_queue(self, spider, slot):
from twisted.internet import reactor
if slot.latercall and slot.latercall.active():
return
@ -143,7 +148,9 @@ class Downloader:
if delay:
penalty = delay - now + slot.lastseen
if penalty > 0:
slot.latercall = reactor.callLater(penalty, self._process_queue, spider, slot)
slot.latercall = reactor.callLater(
penalty, self._process_queue, spider, slot
)
return
# Process enqueued requests if there are free slots to transfer for this slot
@ -166,11 +173,14 @@ class Downloader:
# 2. Notify response_downloaded listeners about the recent download
# before querying queue for next request
def _downloaded(response):
self.signals.send_catch_log(signal=signals.response_downloaded,
response=response,
request=request,
spider=spider)
self.signals.send_catch_log(
signal=signals.response_downloaded,
response=response,
request=request,
spider=spider,
)
return response
dfd.addCallback(_downloaded)
# 3. After response arrives, remove the request from transferring
@ -182,9 +192,9 @@ class Downloader:
def finish_transferring(_):
slot.transferring.remove(request)
self._process_queue(spider, slot)
self.signals.send_catch_log(signal=signals.request_left_downloader,
request=request,
spider=spider)
self.signals.send_catch_log(
signal=signals.request_left_downloader, request=request, spider=spider
)
return _
return dfd.addBoth(finish_transferring)

53
scrapy/core/downloader/contextfactory.py
View File

@ -2,13 +2,22 @@ import warnings
from OpenSSL import SSL
from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.internet.ssl import (
AcceptableCiphers,
CertificateOptions,
optionsForClientTLS,
platformTrust,
)
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
from scrapy.core.downloader.tls import (
DEFAULT_CIPHERS,
ScrapyClientTLSOptions,
openssl_methods,
)
from scrapy.utils.misc import create_instance, load_object
@ -24,7 +33,14 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
def __init__(
self,
method=SSL.SSLv23_METHOD,
tls_verbose_logging=False,
tls_ciphers=None,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
@ -35,9 +51,15 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
@classmethod
def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
tls_verbose_logging = settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING")
tls_ciphers = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
return cls(
method=method,
tls_verbose_logging=tls_verbose_logging,
tls_ciphers=tls_ciphers,
*args,
**kwargs,
)
def getCertificateOptions(self):
# setting verify=True will require you to provide CAs
@ -53,7 +75,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
method=getattr(self, "method", getattr(self, "_ssl_method", None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
)
@ -61,11 +83,16 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# kept for old-style HTTP/1.0 downloader context twisted calls,
# e.g. connectSSL()
def getContext(self, hostname=None, port=None):
return self.getCertificateOptions().getContext()
ctx = self.getCertificateOptions().getContext()
ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT
return ctx
def creatorForNetloc(self, hostname, port):
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
verbose_logging=self.tls_verbose_logging)
return ScrapyClientTLSOptions(
hostname.decode("ascii"),
self.getContext(),
verbose_logging=self.tls_verbose_logging,
)
@implementer(IPolicyForHTTPS)
@ -95,7 +122,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
return optionsForClientTLS(
hostname=hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
extraCertificateOptions={"method": self._ssl_method},
)
@ -118,8 +145,8 @@ class AcceptableProtocolsContextFactory:
def load_context_factory_from_settings(settings, crawler):
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
# try method-aware context factory
try:
context_factory = create_instance(

24
scrapy/core/downloader/handlers/__init__.py
View File

@ -10,19 +10,18 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)
class DownloadHandlers:
def __init__(self, crawler):
self._crawler = crawler
self._schemes = {} # stores acceptable schemes on instancing
self._handlers = {} # stores instanced handlers for schemes
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
)
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
@ -38,7 +37,7 @@ class DownloadHandlers:
if scheme in self._notconfigured:
return None
if scheme not in self._schemes:
self._notconfigured[scheme] = 'no handler available for that scheme'
self._notconfigured[scheme] = "no handler available for that scheme"
return None
return self._load_handler(scheme)
@ -47,7 +46,7 @@ class DownloadHandlers:
path = self._schemes[scheme]
try:
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, 'lazy', True):
if skip_lazy and getattr(dhcls, "lazy", True):
return None
dh = create_instance(
objcls=dhcls,
@ -58,9 +57,12 @@ class DownloadHandlers:
self._notconfigured[scheme] = str(ex)
return None
except Exception as ex:
logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True, extra={'crawler': self._crawler})
logger.error(
'Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True,
extra={"crawler": self._crawler},
)
self._notconfigured[scheme] = str(ex)
return None
else:
@ -71,11 +73,13 @@ class DownloadHandlers:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
raise NotSupported(
f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
)
return handler.download_request(request, spider)
@defer.inlineCallbacks
def _close(self, *_a, **_kw):
for dh in self._handlers.values():
if hasattr(dh, 'close'):
if hasattr(dh, "close"):
yield dh.close()

7
scrapy/core/downloader/handlers/datauri.py
View File

@ -14,9 +14,8 @@ class DataURIDownloadHandler:
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse)
and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
charset = uri.media_type_parameters.get("charset")
resp_kwargs["encoding"] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)

5
scrapy/core/downloader/handlers/file.py
View File

@ -1,3 +1,5 @@
from pathlib import Path
from w3lib.url import file_uri_to_path
from scrapy.responsetypes import responsetypes
@ -10,7 +12,6 @@ class FileDownloadHandler:
@defers
def download_request(self, request, spider):
filepath = file_uri_to_path(request.url)
with open(filepath, 'rb') as fo:
body = fo.read()
body = Path(filepath).read_bytes()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

22
scrapy/core/downloader/handlers/ftp.py
View File

@ -71,9 +71,9 @@ class FTPDownloadHandler:
}
def __init__(self, settings):
self.default_user = settings['FTP_USER']
self.default_password = settings['FTP_PASSWORD']
self.passive_mode = settings['FTP_PASSIVE_MODE']
self.default_user = settings["FTP_USER"]
self.default_password = settings["FTP_PASSWORD"]
self.passive_mode = settings["FTP_PASSIVE_MODE"]
@classmethod
def from_crawler(cls, crawler):
@ -81,12 +81,16 @@ class FTPDownloadHandler:
def download_request(self, request, spider):
from twisted.internet import reactor
parsed_url = urlparse_cached(request)
user = request.meta.get("ftp_user", self.default_user)
password = request.meta.get("ftp_password", self.default_password)
passive_mode = 1 if bool(request.meta.get("ftp_passive",
self.passive_mode)) else 0
creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
passive_mode = (
1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0
)
creator = ClientCreator(
reactor, FTPClient, user, password, passive=passive_mode
)
dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
@ -103,7 +107,7 @@ class FTPDownloadHandler:
def _build_response(self, result, request, protocol):
self.result = result
protocol.close()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
headers = {"local filename": protocol.filename or "", "size": protocol.size}
body = to_bytes(protocol.filename or protocol.body.read())
respcls = responsetypes.from_args(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
@ -115,5 +119,7 @@ class FTPDownloadHandler:
if m:
ftpcode = m.group()
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
return Response(url=request.url, status=httpcode, body=to_bytes(message))
return Response(
url=request.url, status=httpcode, body=to_bytes(message)
)
raise result.type(result.value)

12
scrapy/core/downloader/handlers/http10.py
View File

@ -8,8 +8,10 @@ class HTTP10DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
self.ClientContextFactory = load_object(
settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
self._settings = settings
self._crawler = crawler
@ -25,13 +27,13 @@ class HTTP10DownloadHandler:
def _connect(self, factory):
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
if factory.scheme == b"https":
client_context_factory = create_instance(
objcls=self.ClientContextFactory,
settings=self._settings,
crawler=self._crawler,
)
return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)
return reactor.connectTCP(host, port, factory)

294
scrapy/core/downloader/handlers/http11.py
View File

@ -3,7 +3,6 @@
import ipaddress
import logging
import re
import warnings
from contextlib import suppress
from io import BytesIO
from time import time
@ -13,21 +12,26 @@ from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.python.failure import Failure
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import (
URI,
Agent,
HTTPConnectionPool,
ResponseDone,
ResponseFailed,
)
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.exceptions import StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.python import to_bytes, to_unicode
logger = logging.getLogger(__name__)
@ -38,14 +42,17 @@ class HTTP11DownloadHandler:
self._crawler = crawler
from twisted.internet import reactor
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool.maxPersistentPerHost = settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self._pool._factory.noisy = False
self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE")
self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
self._disconnect_timeout = 1
@classmethod
@ -57,8 +64,8 @@ class HTTP11DownloadHandler:
agent = ScrapyAgent(
contextFactory=self._contextFactory,
pool=self._pool,
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
fail_on_dataloss=self._fail_on_dataloss,
crawler=self._crawler,
)
@ -66,6 +73,7 @@ class HTTP11DownloadHandler:
def close(self):
from twisted.internet import reactor
d = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
@ -98,11 +106,23 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
with this endpoint comes from the pool and a CONNECT has already been issued
for it.
"""
_truncatedLength = 1000
_responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
_responseAnswer = (
r"HTTP/1\.. (?P<status>\d{3})(?P<reason>.{," + str(_truncatedLength) + r"})"
)
_responseMatcher = re.compile(_responseAnswer.encode())
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
def __init__(
self,
reactor,
host,
port,
proxyConf,
contextFactory,
timeout=30,
bindAddress=None,
):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
@ -113,7 +133,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol):
"""Asks the proxy to open a tunnel."""
tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)
tunnelReq = tunnel_request_data(
self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
@ -131,24 +153,30 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
# from the proxy so that we don't send those bytes to the TLS layer
#
# see https://github.com/scrapy/scrapy/issues/2491
if b'\r\n\r\n' not in self._connectBuffer:
if b"\r\n\r\n" not in self._connectBuffer:
return
self._protocol.dataReceived = self._protocolDataReceived
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group('status')) == 200:
if respm and int(respm.group("status")) == 200:
# set proper Server Name Indication extension
sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
sslOptions = self._contextFactory.creatorForNetloc(
self._tunneledHost, self._tunneledPort
)
self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
if respm:
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
extra = {
"status": int(respm.group("status")),
"reason": respm.group("reason").strip(),
}
else:
extra = rcvd_bytes[:self._truncatedLength]
extra = rcvd_bytes[: self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')
TunnelError(
"Could not open CONNECT tunnel with proxy "
f"{self._host}:{self._port} [{extra!r}]"
)
)
def connectFailed(self, reason):
@ -175,12 +203,12 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
>>> s(tunnel_request_data(b"example.com", "8090"))
'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
"""
host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
tunnel_req += b'Host: ' + host_value + b'\r\n'
host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
tunnel_req += b"Host: " + host_value + b"\r\n"
if proxy_auth_header:
tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
tunnel_req += b'\r\n'
tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n"
tunnel_req += b"\r\n"
return tunnel_req
@ -192,8 +220,15 @@ class TunnelingAgent(Agent):
proxy involved.
"""
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
def __init__(
self,
reactor,
proxyConf,
contextFactory=None,
connectTimeout=None,
bindAddress=None,
pool=None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@ -209,7 +244,9 @@ class TunnelingAgent(Agent):
bindAddress=self._endpointFactory._bindAddress,
)
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
def _requestWithEndpoint(
self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
):
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
@ -226,8 +263,9 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
def __init__(
self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None
):
super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
@ -259,8 +297,17 @@ class ScrapyAgent:
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
def __init__(
self,
contextFactory=None,
connectTimeout=10,
bindAddress=None,
pool=None,
maxsize=0,
warnsize=0,
fail_on_dataloss=True,
crawler=None,
):
self._contextFactory = contextFactory
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
@ -273,24 +320,15 @@ class ScrapyAgent:
def _get_agent(self, request, timeout):
from twisted.internet import reactor
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
bindaddress = request.meta.get("bindaddress") or self._bindAddress
proxy = request.meta.get("proxy")
if proxy:
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Zyte Smart Proxy Manager, it doesn't require "
"this mode anymore, so you should update scrapy-crawlera "
"to scrapy-zyte-smartproxy and remove '?noconnect' "
"from the Zyte Smart Proxy Manager URL.",
ScrapyDeprecationWarning,
)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
if scheme == b"https":
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
return self._TunnelingAgent(
reactor=reactor,
@ -300,18 +338,15 @@ class ScrapyAgent:
bindAddress=bindaddress,
pool=self._pool,
)
else:
proxyScheme = proxyScheme or b'http'
proxyHost = to_bytes(proxyHost, encoding='ascii')
proxyPort = to_bytes(str(proxyPort), encoding='ascii')
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
proxyScheme = proxyScheme or b"http"
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", ""))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding="ascii"),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
return self._Agent(
reactor=reactor,
@ -323,7 +358,8 @@ class ScrapyAgent:
def download_request(self, request):
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connectTimeout
timeout = request.meta.get("download_timeout") or self._connectTimeout
agent = self._get_agent(request, timeout)
# request details
@ -331,13 +367,15 @@ class ScrapyAgent:
method = to_bytes(request.method)
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
headers.removeHeader(b'Proxy-Authorization')
headers.removeHeader(b"Proxy-Authorization")
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
else:
bodyproducer = None
start_time = time()
d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
d = agent.request(
method, to_bytes(url, encoding="ascii"), headers, bodyproducer
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
@ -360,14 +398,14 @@ class ScrapyAgent:
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time
request.meta["download_latency"] = time() - start_time
return result
@staticmethod
def _headers_from_twisted_response(response):
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b'Content-Length'] = str(response.length).encode()
headers[b"Content-Length"] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
@ -381,8 +419,10 @@ class ScrapyAgent:
)
for handler, result in headers_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__})
logger.debug(
"Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__},
)
txresponse._transport.stopProducing()
txresponse._transport.loseConnection()
return {
@ -404,15 +444,23 @@ class ScrapyAgent:
"ip_address": None,
}
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
maxsize = request.meta.get("download_maxsize", self._maxsize)
warnsize = request.meta.get("download_warnsize", self._warnsize)
expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
fail_on_dataloss = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if maxsize and expected_size > maxsize:
warning_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
warning_msg = (
"Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s)."
)
warning_args = {
"url": request.url,
"size": expected_size,
"maxsize": maxsize,
}
logger.warning(warning_msg, warning_args)
@ -420,9 +468,11 @@ class ScrapyAgent:
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
"download warn size (%(warnsize)s) in request %(request)s.",
{'size': expected_size, 'warnsize': warnsize, 'request': request})
logger.warning(
"Expected response size (%(size)s) larger than "
"download warn size (%(warnsize)s) in request %(request)s.",
{"size": expected_size, "warnsize": warnsize, "request": request},
)
def _cancel(_):
# Abort connection immediately.
@ -472,7 +522,6 @@ class ScrapyAgent:
@implementer(IBodyProducer)
class _RequestBodyProducer:
def __init__(self, body):
self.body = body
self.length = len(body)
@ -489,8 +538,16 @@ class _RequestBodyProducer:
class _ResponseReader(protocol.Protocol):
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
def __init__(
self,
finished,
txresponse,
request,
maxsize,
warnsize,
fail_on_dataloss,
crawler,
):
self._finished = finished
self._txresponse = txresponse
self._request = request
@ -506,22 +563,28 @@ class _ResponseReader(protocol.Protocol):
self._crawler = crawler
def _finish_response(self, flags=None, failure=None):
self._finished.callback({
"txresponse": self._txresponse,
"body": self._bodybuf.getvalue(),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"failure": failure,
})
self._finished.callback(
{
"txresponse": self._txresponse,
"body": self._bodybuf.getvalue(),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"failure": failure,
}
)
def connectionMade(self):
if self._certificate is None:
with suppress(AttributeError):
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
self._certificate = ssl.Certificate(
self.transport._producer.getPeerCertificate()
)
if self._ip_address is None:
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
self._ip_address = ipaddress.ip_address(
self.transport._producer.getPeer().host
)
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered data.
@ -539,29 +602,40 @@ class _ResponseReader(protocol.Protocol):
)
for handler, result in bytes_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
logger.debug(
"Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__},
)
self.transport.stopProducing()
self.transport.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.warning("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
logger.warning(
"Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{
"bytes": self._bytes_received,
"maxsize": self._maxsize,
"request": self._request,
},
)
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
if (
self._warnsize
and self._bytes_received > self._warnsize
and not self._reached_warnsize
):
self._reached_warnsize = True
logger.warning("Received more bytes than download "
"warn size (%(warnsize)s) in request %(request)s.",
{'warnsize': self._warnsize,
'request': self._request})
logger.warning(
"Received more bytes than download "
"warn size (%(warnsize)s) in request %(request)s.",
{"warnsize": self._warnsize, "request": self._request},
)
def connectionLost(self, reason):
if self._finished.called:
@ -575,16 +649,20 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["partial"])
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if reason.check(ResponseFailed) and any(
r.check(_DataLoss) for r in reason.value.reasons
):
if not self._fail_on_dataloss:
self._finish_response(flags=["dataloss"])
return
elif not self._fail_on_dataloss_warned:
logger.warning("Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",
self._txresponse.request.absoluteURI.decode())
if not self._fail_on_dataloss_warned:
logger.warning(
"Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",
self._txresponse.request.absoluteURI.decode(),
)
self._fail_on_dataloss_warned = True
self._finished.errback(reason)

51
scrapy/core/downloader/handlers/http2.py
View File

@ -1,4 +1,3 @@
import warnings
from time import time
from typing import Optional, Type, TypeVar
from urllib.parse import urldefrag
@ -17,8 +16,9 @@ from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
H2DownloadHandlerOrSubclass = TypeVar(
"H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
)
class H2DownloadHandler:
@ -26,11 +26,14 @@ class H2DownloadHandler:
self._crawler = crawler
from twisted.internet import reactor
self._pool = H2ConnectionPool(reactor, settings)
self._context_factory = load_context_factory_from_settings(settings, crawler)
@classmethod
def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
def from_crawler(
cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler
) -> H2DownloadHandlerOrSubclass:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
@ -50,7 +53,8 @@ class ScrapyH2Agent:
_ProxyAgent = ScrapyProxyH2Agent
def __init__(
self, context_factory,
self,
context_factory,
pool: H2ConnectionPool,
connect_timeout: int = 10,
bind_address: Optional[bytes] = None,
@ -64,30 +68,22 @@ class ScrapyH2Agent:
def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
from twisted.internet import reactor
bind_address = request.meta.get('bindaddress') or self._bind_address
proxy = request.meta.get('proxy')
bind_address = request.meta.get("bindaddress") or self._bind_address
proxy = request.meta.get("proxy")
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = proxy_host.decode()
omit_connect_tunnel = b'noconnect' in proxy_params
if omit_connect_tunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is not "
"supported by the downloader handler. If you use Zyte "
"Smart Proxy Manager, it doesn't require this mode "
"anymore, so you should update scrapy-crawlera to "
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
"Zyte Smart Proxy Manager URL."
)
if scheme == b'https' and not omit_connect_tunnel:
if scheme == b"https":
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
raise NotImplementedError(
"Tunneling via CONNECT method using HTTP/2.0 is not yet supported"
)
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
@ -103,7 +99,8 @@ class ScrapyH2Agent:
def download_request(self, request: Request, spider: Spider) -> Deferred:
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connect_timeout
timeout = request.meta.get("download_timeout") or self._connect_timeout
agent = self._get_agent(request, timeout)
start_time = time()
@ -115,12 +112,16 @@ class ScrapyH2Agent:
return d
@staticmethod
def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
request.meta['download_latency'] = time() - start_time
def _cb_latency(
response: Response, request: Request, start_time: float
) -> Response:
request.meta["download_latency"] = time() - start_time
return response
@staticmethod
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
def _cb_timeout(
response: Response, request: Request, timeout: float, timeout_cl: DelayedCall
) -> Response:
if timeout_cl.active():
timeout_cl.cancel()
return response

60
scrapy/core/downloader/handlers/s3.py
View File

@ -6,40 +6,49 @@ from scrapy.utils.misc import create_instance
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
def __init__(
self,
settings,
*,
crawler=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler,
**kw,
):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
raise NotConfigured("missing botocore library")
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
aws_access_key_id = settings["AWS_ACCESS_KEY_ID"]
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"]
if not aws_session_token:
aws_session_token = settings['AWS_SESSION_TOKEN']
aws_session_token = settings["AWS_SESSION_TOKEN"]
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
# unless 'anon' was set explicitly (True/False).
anon = kw.get('anon')
anon = kw.get("anon")
if anon is None and not aws_access_key_id and not aws_secret_access_key:
kw['anon'] = True
self.anon = kw.get('anon')
kw["anon"] = True
self.anon = kw.get("anon")
self._signer = None
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
kw.pop("anon", None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
raise TypeError(f"Unexpected keyword arguments: {kw}")
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key, aws_session_token))
SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
self._signer = SignerCls(
botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key, aws_session_token
)
)
_http_handler = create_instance(
objcls=httpdownloadhandler,
@ -54,20 +63,21 @@ class S3DownloadHandler:
def download_request(self, request, spider):
p = urlparse_cached(request)
scheme = 'https' if request.meta.get('is_secure') else 'http'
scheme = "https" if request.meta.get("is_secure") else "http"
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
path = p.path + "?" + p.query if p.query else p.path
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
if self.anon:
request = request.replace(url=url)
else:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
headers=request.headers.to_unicode_dict(),
data=request.body)
data=request.body,
)
self._signer.add_auth(awsrequest)
request = request.replace(
url=url, headers=awsrequest.headers.items())
request = request.replace(url=url, headers=awsrequest.headers.items())
return self._download_http(request, spider)

45
scrapy/core/downloader/middleware.py
View File

@ -12,34 +12,37 @@ from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
class DownloaderMiddlewareManager(MiddlewareManager):
component_name = 'downloader middleware'
component_name = "downloader middleware"
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(
settings.getwithbase('DOWNLOADER_MIDDLEWARES'))
return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES"))
def _add_middleware(self, mw):
if hasattr(mw, 'process_request'):
self.methods['process_request'].append(mw.process_request)
if hasattr(mw, 'process_response'):
self.methods['process_response'].appendleft(mw.process_response)
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].appendleft(mw.process_exception)
if hasattr(mw, "process_request"):
self.methods["process_request"].append(mw.process_request)
if hasattr(mw, "process_response"):
self.methods["process_response"].appendleft(mw.process_response)
if hasattr(mw, "process_exception"):
self.methods["process_exception"].appendleft(mw.process_exception)
def download(self, download_func: Callable, request: Request, spider: Spider):
@defer.inlineCallbacks
def process_request(request: Request):
for method in self.methods['process_request']:
for method in self.methods["process_request"]:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
response = yield deferred_from_coro(
method(request=request, spider=spider)
)
if response is not None and not isinstance(
response, (Response, Request)
):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {response.__class__.__name__}"
@ -55,9 +58,11 @@ class DownloaderMiddlewareManager(MiddlewareManager):
elif isinstance(response, Request):
return response
for method in self.methods['process_response']:
for method in self.methods["process_response"]:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
response = yield deferred_from_coro(
method(request=request, response=response, spider=spider)
)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return Response or Request, "
@ -70,10 +75,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
@defer.inlineCallbacks
def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
for method in self.methods["process_exception"]:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
response = yield deferred_from_coro(
method(request=request, exception=exception, spider=spider)
)
if response is not None and not isinstance(
response, (Response, Request)
):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {type(response)}"

59
scrapy/core/downloader/tls.py
View File

@ -2,26 +2,29 @@ import logging
from OpenSSL import SSL
from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
from twisted.internet._sslverify import (
ClientTLSOptions,
VerificationError,
verifyHostname,
)
from twisted.internet.ssl import AcceptableCiphers
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
from scrapy.utils.ssl import get_temp_key_info, x509name_to_string
logger = logging.getLogger(__name__)
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
METHOD_TLSv11 = 'TLSv1.1'
METHOD_TLSv12 = 'TLSv1.2'
METHOD_TLS = "TLS"
METHOD_TLSv10 = "TLSv1.0"
METHOD_TLSv11 = "TLSv1.1"
METHOD_TLSv12 = "TLSv1.2"
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only
METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only
}
@ -45,32 +48,38 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
connection.set_tlsext_host_name(self._hostnameBytes)
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
logger.debug(
"SSL connection to %s using protocol %s, cipher %s",
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),
x509name_to_string(server_cert.get_subject()),
)
logger.debug(
'SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),
x509name_to_string(server_cert.get_subject()),
)
key_info = get_temp_key_info(connection._ssl)
if key_info:
logger.debug('SSL temp key: %s', key_info)
logger.debug("SSL temp key: %s", key_info)
try:
verifyHostname(connection, self._hostnameASCII)
except (CertificateError, VerificationError) as e:
logger.warning(
'Remote certificate is not valid for hostname "%s"; %s',
self._hostnameASCII, e)
self._hostnameASCII,
e,
)
except ValueError as e:
logger.warning(
'Ignoring error while verifying certificate '
"Ignoring error while verifying certificate "
'from host "%s" (exception: %r)',
self._hostnameASCII, e)
self._hostnameASCII,
e,
)
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")

66
scrapy/core/downloader/webclient.py
View File

@ -1,47 +1,47 @@
import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from urllib.parse import urldefrag, urlparse, urlunparse
from twisted.web.http import HTTPClient
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from twisted.web.http import HTTPClient
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
# Assume parsed is urlparse-d from Request.url,
# which was passed via safe_url_string and is ascii-only.
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, ""))
path = to_bytes(path, encoding="ascii")
host = to_bytes(parsed.hostname, encoding="ascii")
port = parsed.port
scheme = to_bytes(parsed.scheme, encoding="ascii")
netloc = to_bytes(parsed.netloc, encoding="ascii")
if port is None:
port = 443 if scheme == b'https' else 80
port = 443 if scheme == b"https" else 80
return scheme, netloc, host, port, path
def _parse(url):
""" Return tuple of (scheme, netloc, host, port, path),
"""Return tuple of (scheme, netloc, host, port, path),
all in bytes except for port which is int.
Assume url is from Request.url, which was passed via safe_url_string
and is ascii-only.
"""
url = url.strip()
if not re.match(r'^\w+://', url):
url = '//' + url
if not re.match(r"^\w+://", url):
url = "//" + url
parsed = urlparse(url)
return _parsed_url_args(parsed)
class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
delimiter = b"\n"
def connectionMade(self):
self.headers = Headers() # bucket for response headers
@ -75,8 +75,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.factory.noPage(reason)
def handleResponse(self, response):
if self.factory.method.upper() == b'HEAD':
self.factory.page(b'')
if self.factory.method.upper() == b"HEAD":
self.factory.page(b"")
elif self.length is not None and self.length > 0:
self.factory.noPage(self._connection_lost_reason)
else:
@ -87,18 +87,21 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.loseConnection()
# transport cleanup needed for HTTPS connections
if self.factory.url.startswith(b'https'):
if self.factory.url.startswith(b"https"):
self.transport.stopProducing()
self.factory.noPage(
defer.TimeoutError(f"Getting {self.factory.url} took longer "
f"than {self.factory.timeout} seconds."))
defer.TimeoutError(
f"Getting {self.factory.url} took longer "
f"than {self.factory.timeout} seconds."
)
)
# This class used to inherit from Twisted’s
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
# non-overriden code into this class.
# non-overridden code into this class.
class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
@ -109,16 +112,24 @@ class ScrapyHTTPClientFactory(ClientFactory):
afterFoundGet = False
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
request.meta["download_latency"] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
return respcls(
url=self._url,
status=status,
headers=headers,
body=body,
protocol=to_unicode(self.version),
)
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(
parsed
)
proxy = request.meta.get("proxy")
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
@ -126,12 +137,12 @@ class ScrapyHTTPClientFactory(ClientFactory):
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
self.url = to_bytes(self._url, encoding='ascii')
self.method = to_bytes(request.method, encoding='ascii')
self.url = to_bytes(self._url, encoding="ascii")
self.method = to_bytes(request.method, encoding="ascii")
self.body = request.body or None
self.headers = Headers(request.headers)
self.response_headers = None
self.timeout = request.meta.get('download_timeout') or timeout
self.timeout = request.meta.get("download_timeout") or timeout
self.start_time = time()
self.deferred = defer.Deferred().addCallback(self._build_response, request)
@ -146,16 +157,16 @@ class ScrapyHTTPClientFactory(ClientFactory):
self._set_connection_attributes(request)
# set Host header based on url
self.headers.setdefault('Host', self.netloc)
self.headers.setdefault("Host", self.netloc)
# set Content-Length based len of body
if self.body is not None:
self.headers['Content-Length'] = len(self.body)
self.headers["Content-Length"] = len(self.body)
# just in case a broken http/1.1 decides to keep connection alive
self.headers.setdefault("Connection", "close")
# Content-Length must be specified in POST method even with no body
elif self.method == b'POST':
self.headers['Content-Length'] = 0
elif self.method == b"POST":
self.headers["Content-Length"] = 0
def __repr__(self):
return f"<{self.__class__.__name__}: {self.url}>"
@ -171,6 +182,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
p.afterFoundGet = self.afterFoundGet
if self.timeout:
from twisted.internet import reactor
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p
@ -185,7 +197,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
@param version: The HTTP version.
@type version: L{bytes}
@param status: The HTTP status code, an integer represented as a
bytestring.
bytestring.
@type status: L{bytes}
@param message: The HTTP status message.
@type message: L{bytes}

165
scrapy/core/engine.py
View File

@ -15,19 +15,14 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.scraper import Scraper
from scrapy.exceptions import (
CloseSpider,
DontCloseSpider,
ScrapyDeprecationWarning,
)
from scrapy.http import Response, Request
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.reactor import CallLaterOnce
logger = logging.getLogger(__name__)
@ -79,13 +74,14 @@ class ExecutionEngine:
self.running = False
self.paused = False
self.scheduler_cls = self._get_scheduler_class(crawler.settings)
downloader_cls = load_object(self.settings['DOWNLOADER'])
downloader_cls = load_object(self.settings["DOWNLOADER"])
self.downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
def _get_scheduler_class(self, settings: BaseSettings) -> type:
from scrapy.core.scheduler import BaseScheduler
scheduler_cls = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
@ -106,6 +102,7 @@ class ExecutionEngine:
def stop(self) -> Deferred:
"""Gracefully stop the execution engine"""
@inlineCallbacks
def _finish_stopping_engine(_) -> Deferred:
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
@ -115,7 +112,11 @@ class ExecutionEngine:
raise RuntimeError("Engine not running")
self.running = False
dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None)
dfd = (
self.close_spider(self.spider, reason="shutdown")
if self.spider is not None
else succeed(None)
)
return dfd.addBoth(_finish_stopping_engine)
def close(self) -> Deferred:
@ -126,7 +127,9 @@ class ExecutionEngine:
if self.running:
return self.stop() # will also close spider and downloader
if self.spider is not None:
return self.close_spider(self.spider, reason="shutdown") # will also close downloader
return self.close_spider(
self.spider, reason="shutdown"
) # will also close downloader
return succeed(self.downloader.close())
def pause(self) -> None:
@ -144,7 +147,10 @@ class ExecutionEngine:
if self.paused:
return None
while not self._needs_backout() and self._next_request_from_scheduler() is not None:
while (
not self._needs_backout()
and self._next_request_from_scheduler() is not None
):
pass
if self.slot.start_requests is not None and not self._needs_backout():
@ -154,7 +160,11 @@ class ExecutionEngine:
self.slot.start_requests = None
except Exception:
self.slot.start_requests = None
logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider})
logger.error(
"Error while obtaining start requests",
exc_info=True,
extra={"spider": self.spider},
)
else:
self.crawl(request)
@ -179,18 +189,30 @@ class ExecutionEngine:
d = self._download(request, self.spider)
d.addBoth(self._handle_downloader_output, request)
d.addErrback(lambda f: logger.info('Error while handling downloader output',
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider}))
d.addErrback(
lambda f: logger.info(
"Error while handling downloader output",
exc_info=failure_to_exc_info(f),
extra={"spider": self.spider},
)
)
d.addBoth(lambda _: self.slot.remove_request(request))
d.addErrback(lambda f: logger.info('Error while removing request from slot',
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider}))
d.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
exc_info=failure_to_exc_info(f),
extra={"spider": self.spider},
)
)
slot = self.slot
d.addBoth(lambda _: slot.nextcall.schedule())
d.addErrback(lambda f: logger.info('Error while scheduling new request',
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider}))
d.addErrback(
lambda f: logger.info(
"Error while scheduling new request",
exc_info=failure_to_exc_info(f),
extra={"spider": self.spider},
)
)
return d
def _handle_downloader_output(
@ -199,7 +221,9 @@ class ExecutionEngine:
assert self.spider is not None # typing
if not isinstance(result, (Request, Response, Failure)):
raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
raise TypeError(
f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}"
)
# downloader middleware can return requests (for example, redirects)
if isinstance(result, Request):
@ -211,7 +235,7 @@ class ExecutionEngine:
lambda f: logger.error(
"Error while enqueuing downloader output",
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider},
extra={"spider": self.spider},
)
)
return d
@ -244,16 +268,22 @@ class ExecutionEngine:
stacklevel=2,
)
if spider is not self.spider:
raise RuntimeError(f"The spider {spider.name!r} does not match the open spider")
raise RuntimeError(
f"The spider {spider.name!r} does not match the open spider"
)
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
self._schedule_request(request, self.spider)
self.slot.nextcall.schedule() # type: ignore[union-attr]
def _schedule_request(self, request: Request, spider: Spider) -> None:
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
self.signals.send_catch_log(
signals.request_scheduled, request=request, spider=spider
)
if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
self.signals.send_catch_log(
signals.request_dropped, request=request, spider=spider
)
def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
@ -264,10 +294,14 @@ class ExecutionEngine:
stacklevel=2,
)
if spider is not self.spider:
logger.warning("The spider '%s' does not match the open spider", spider.name)
logger.warning(
"The spider '%s' does not match the open spider", spider.name
)
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
return self._download(request, spider).addBoth(self._downloaded, request, spider)
return self._download(request, spider).addBoth(
self._downloaded, request, spider
)
def _downloaded(
self, result: Union[Response, Request], request: Request, spider: Spider
@ -286,7 +320,9 @@ class ExecutionEngine:
def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
if not isinstance(result, (Response, Request)):
raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
raise TypeError(
f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}"
)
if isinstance(result, Response):
if result.request is None:
result.request = request
@ -311,13 +347,19 @@ class ExecutionEngine:
return dwld
@inlineCallbacks
def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True):
def open_spider(
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
):
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={'spider': spider})
logger.info("Spider opened", extra={"spider": spider})
nextcall = CallLaterOnce(self._next_request)
scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler)
start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
scheduler = create_instance(
self.scheduler_cls, settings=None, crawler=self.crawler
)
start_requests = yield self.scraper.spidermw.process_start_requests(
start_requests, spider
)
self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.spider = spider
if hasattr(scheduler, "open"):
@ -337,7 +379,9 @@ class ExecutionEngine:
"""
assert self.spider is not None # typing
expected_ex = (DontCloseSpider, CloseSpider)
res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
res = self.signals.send_catch_log(
signals.spider_idle, spider=self.spider, dont_log=expected_ex
)
detected_ex = {
ex: x.value
for _, x in res
@ -347,7 +391,7 @@ class ExecutionEngine:
if DontCloseSpider in detected_ex:
return None
if self.spider_is_idle():
ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished'))
ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished"))
assert isinstance(ex, CloseSpider) # typing
self.close_spider(self.spider, reason=ex.reason)
@ -359,40 +403,55 @@ class ExecutionEngine:
if self.slot.closing is not None:
return self.slot.closing
logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})
logger.info(
"Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
)
dfd = self.slot.close()
def log_failure(msg: str) -> Callable:
def errback(failure: Failure) -> None:
logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
logger.error(
msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider}
)
return errback
dfd.addBoth(lambda _: self.downloader.close())
dfd.addErrback(log_failure('Downloader close failure'))
dfd.addErrback(log_failure("Downloader close failure"))
dfd.addBoth(lambda _: self.scraper.close_spider(spider))
dfd.addErrback(log_failure('Scraper close failure'))
dfd.addErrback(log_failure("Scraper close failure"))
if hasattr(self.slot.scheduler, "close"):
dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
signal=signals.spider_closed, spider=spider, reason=reason,
))
dfd.addErrback(log_failure('Error while sending spider_close signal'))
dfd.addBoth(
lambda _: self.signals.send_catch_log_deferred(
signal=signals.spider_closed,
spider=spider,
reason=reason,
)
)
dfd.addErrback(log_failure("Error while sending spider_close signal"))
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
dfd.addErrback(log_failure('Stats close failure'))
dfd.addErrback(log_failure("Stats close failure"))
dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))
dfd.addBoth(
lambda _: logger.info(
"Spider closed (%(reason)s)",
{"reason": reason},
extra={"spider": spider},
)
)
dfd.addBoth(lambda _: setattr(self, 'slot', None))
dfd.addErrback(log_failure('Error while unassigning slot'))
dfd.addBoth(lambda _: setattr(self, "slot", None))
dfd.addErrback(log_failure("Error while unassigning slot"))
dfd.addBoth(lambda _: setattr(self, 'spider', None))
dfd.addErrback(log_failure('Error while unassigning spider'))
dfd.addBoth(lambda _: setattr(self, "spider", None))
dfd.addErrback(log_failure("Error while unassigning spider"))
dfd.addBoth(lambda _: self._spider_closed_callback(spider))
@ -408,7 +467,11 @@ class ExecutionEngine:
return [self.spider] if self.spider is not None else []
def has_capacity(self) -> bool:
warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2)
warnings.warn(
"ExecutionEngine.has_capacity is deprecated",
ScrapyDeprecationWarning,
stacklevel=2,
)
return not bool(self.slot)
def schedule(self, request: Request, spider: Spider) -> None:

18
scrapy/core/http2/agent.py
View File

@ -10,7 +10,7 @@ from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpoint
from twisted.web.error import SchemeNotSupported
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
from scrapy.http.request import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
@ -28,7 +28,9 @@ class H2ConnectionPool:
# Save all requests that arrive before the connection is established
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
def get_connection(
self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
) -> Deferred:
if key in self._pending_requests:
# Received a request while connecting to remote
# Create a deferred which will fire with the H2ClientProtocol
@ -46,7 +48,9 @@ class H2ConnectionPool:
# No connection is established for the given URI
return self._new_connection(key, uri, endpoint)
def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
def _new_connection(
self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
) -> Deferred:
self._pending_requests[key] = deque()
conn_lost_deferred = Deferred()
@ -102,7 +106,9 @@ class H2Agent:
) -> None:
self._reactor = reactor
self._pool = pool
self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
self._context_factory = AcceptableProtocolsContextFactory(
context_factory, acceptable_protocols=[b"h2"]
)
self.endpoint_factory = _StandardEndpointFactory(
self._reactor, self._context_factory, connect_timeout, bind_address
)
@ -118,7 +124,7 @@ class H2Agent:
return uri.scheme, uri.host, uri.port
def request(self, request: Request, spider: Spider) -> Deferred:
uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
uri = URI.fromBytes(bytes(request.url, encoding="utf-8"))
try:
endpoint = self.get_endpoint(uri)
except SchemeNotSupported:
@ -140,7 +146,7 @@ class ScrapyProxyH2Agent(H2Agent):
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
super().__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,

123
scrapy/core/http2/protocol.py
View File

@ -9,15 +9,21 @@ from h2.config import H2Configuration
from h2.connection import H2Connection
from h2.errors import ErrorCodes
from h2.events import (
Event, ConnectionTerminated, DataReceived, ResponseReceived,
SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
WindowUpdated
ConnectionTerminated,
DataReceived,
Event,
ResponseReceived,
SettingsAcknowledged,
StreamEnded,
StreamReset,
UnknownFrameReceived,
WindowUpdated,
)
from h2.exceptions import FrameTooLargeError, H2Error
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
from twisted.internet.protocol import connectionDone, Factory, Protocol
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
from twisted.python.failure import Failure
@ -29,7 +35,6 @@ from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
@ -37,12 +42,11 @@ PROTOCOL_NAME = b"h2"
class InvalidNegotiatedProtocol(H2Error):
def __init__(self, negotiated_protocol: bytes) -> None:
self.negotiated_protocol = negotiated_protocol
def __str__(self) -> str:
return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
return f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}"
class RemoteTerminatedConnection(H2Error):
@ -55,11 +59,13 @@ class RemoteTerminatedConnection(H2Error):
self.terminate_event = event
def __str__(self) -> str:
return f'Received GOAWAY frame from {self.remote_ip_address!r}'
return f"Received GOAWAY frame from {self.remote_ip_address!r}"
class MethodNotAllowed405(H2Error):
def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
def __init__(
self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]
) -> None:
self.remote_ip_address = remote_ip_address
def __str__(self) -> str:
@ -70,7 +76,9 @@ class MethodNotAllowed405(H2Error):
class H2ClientProtocol(Protocol, TimeoutMixin):
IDLE_TIMEOUT = 240
def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
def __init__(
self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
) -> None:
"""
Arguments:
uri -- URI of the base url to which HTTP/2 Connection will be made.
@ -82,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""
self._conn_lost_deferred = conn_lost_deferred
config = H2Configuration(client_side=True, header_encoding='utf-8')
config = H2Configuration(client_side=True, header_encoding="utf-8")
self.conn = H2Connection(config=config)
# ID of the next request stream
@ -105,31 +113,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# initialized when connection is successfully made
self.metadata: Dict = {
# Peer certificate instance
'certificate': None,
"certificate": None,
# Address of the server we are connected to which
# is updated when HTTP/2 connection is made successfully
'ip_address': None,
"ip_address": None,
# URI of the peer HTTP/2 connection is made
'uri': uri,
"uri": uri,
# Both ip_address and uri are used by the Stream before
# initiating the request to verify that the base address
# Variables taken from Project Settings
'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
"default_download_maxsize": settings.getint("DOWNLOAD_MAXSIZE"),
"default_download_warnsize": settings.getint("DOWNLOAD_WARNSIZE"),
# Counter to keep track of opened streams. This counter
# is used to make sure that not more than MAX_CONCURRENT_STREAMS
# streams are opened which leads to ProtocolError
# We use simple FIFO policy to handle pending requests
'active_streams': 0,
"active_streams": 0,
# Flag to keep track if settings were acknowledged by the remote
# This ensures that we have established a HTTP/2 connection
'settings_acknowledged': False,
"settings_acknowledged": False,
}
@property
@ -138,7 +140,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
This is used while initiating pending streams to make sure
that we initiate stream only during active HTTP/2 Connection
"""
return bool(self.transport.connected) and self.metadata['settings_acknowledged']
return bool(self.transport.connected) and self.metadata["settings_acknowledged"]
@property
def allowed_max_concurrent_streams(self) -> int:
@ -149,7 +151,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""
return min(
self.conn.local_settings.max_concurrent_streams,
self.conn.remote_settings.max_concurrent_streams
self.conn.remote_settings.max_concurrent_streams,
)
def _send_pending_requests(self) -> None:
@ -159,37 +161,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""
while (
self._pending_request_stream_pool
and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
and self.metadata["active_streams"] < self.allowed_max_concurrent_streams
and self.h2_connected
):
self.metadata['active_streams'] += 1
self.metadata["active_streams"] += 1
stream = self._pending_request_stream_pool.popleft()
stream.initiate_request()
self._write_to_transport()
def pop_stream(self, stream_id: int) -> Stream:
"""Perform cleanup when a stream is closed
"""
"""Perform cleanup when a stream is closed"""
stream = self.streams.pop(stream_id)
self.metadata['active_streams'] -= 1
self.metadata["active_streams"] -= 1
self._send_pending_requests()
return stream
def _new_stream(self, request: Request, spider: Spider) -> Stream:
"""Instantiates a new Stream object
"""
"""Instantiates a new Stream object"""
stream = Stream(
stream_id=next(self._stream_id_generator),
request=request,
protocol=self,
download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']),
download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']),
download_maxsize=getattr(
spider, "download_maxsize", self.metadata["default_download_maxsize"]
),
download_warnsize=getattr(
spider, "download_warnsize", self.metadata["default_download_warnsize"]
),
)
self.streams[stream.stream_id] = stream
return stream
def _write_to_transport(self) -> None:
""" Write data to the underlying transport connection
"""Write data to the underlying transport connection
from the HTTP2 connection instance if any
"""
# Reset the idle timeout as connection is still actively sending data
@ -200,7 +204,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
def request(self, request: Request, spider: Spider) -> Deferred:
if not isinstance(request, Request):
raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
raise TypeError(
f"Expected scrapy.http.Request, received {request.__class__.__qualname__}"
)
stream = self._new_stream(request, spider)
d = stream.get_response()
@ -221,7 +227,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self.setTimeout(self.IDLE_TIMEOUT)
destination = self.transport.getPeer()
self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
self.metadata["ip_address"] = ipaddress.ip_address(destination.host)
# Initiate H2 Connection
self.conn.initiate_connection()
@ -237,9 +243,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""
Close the connection if it's not made via the expected protocol
"""
if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME:
if (
self.transport.negotiatedProtocol is not None
and self.transport.negotiatedProtocol != PROTOCOL_NAME
):
# we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer
self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)])
self._lose_connection_with_error(
[InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]
)
def _check_received_data(self, data: bytes) -> None:
"""Checks for edge cases where the connection to remote fails
@ -248,8 +259,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
Arguments:
data -- Data received from the remote
"""
if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):
raise MethodNotAllowed405(self.metadata['ip_address'])
if data.startswith(b"HTTP/2.0 405 Method Not Allowed"):
raise MethodNotAllowed405(self.metadata["ip_address"])
def dataReceived(self, data: bytes) -> None:
# Reset the idle timeout as connection is still actively receiving data
@ -284,7 +295,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
if (
self.conn.open_outbound_streams > 0
or self.conn.open_inbound_streams > 0
or self.metadata['active_streams'] > 0
or self.metadata["active_streams"] > 0
):
error_code = ErrorCodes.PROTOCOL_ERROR
else:
@ -292,9 +303,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self.conn.close_connection(error_code=error_code)
self._write_to_transport()
self._lose_connection_with_error([
TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")
])
self._lose_connection_with_error(
[TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
)
def connectionLost(self, reason: Failure = connectionDone) -> None:
"""Called by Twisted when the transport connection is lost.
@ -311,13 +322,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self._conn_lost_deferred.callback(self._conn_lost_errors)
for stream in self.streams.values():
if stream.metadata['request_sent']:
if stream.metadata["request_sent"]:
close_reason = StreamCloseReason.CONNECTION_LOST
else:
close_reason = StreamCloseReason.INACTIVE
stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
self.metadata['active_streams'] -= len(self.streams)
self.metadata["active_streams"] -= len(self.streams)
self.streams.clear()
self._pending_request_stream_pool.clear()
self.conn.close_connection()
@ -345,13 +356,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
elif isinstance(event, SettingsAcknowledged):
self.settings_acknowledged(event)
elif isinstance(event, UnknownFrameReceived):
logger.warning('Unknown frame received: %s', event.frame)
logger.warning("Unknown frame received: %s", event.frame)
# Event handler functions starts here
def connection_terminated(self, event: ConnectionTerminated) -> None:
self._lose_connection_with_error([
RemoteTerminatedConnection(self.metadata['ip_address'], event)
])
self._lose_connection_with_error(
[RemoteTerminatedConnection(self.metadata["ip_address"], event)]
)
def data_received(self, event: DataReceived) -> None:
try:
@ -370,14 +381,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
stream.receive_headers(event.headers)
def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
self.metadata['settings_acknowledged'] = True
self.metadata["settings_acknowledged"] = True
# Send off all the pending requests as now we have
# established a proper HTTP/2 connection
self._send_pending_requests()
# Update certificate when our HTTP/2 connection is established
self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate())
def stream_ended(self, event: StreamEnded) -> None:
try:
@ -406,7 +417,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
@implementer(IProtocolNegotiationFactory)
class H2ClientFactory(Factory):
def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
def __init__(
self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
) -> None:
self.uri = uri
self.settings = settings
self.conn_lost_deferred = conn_lost_deferred

242
scrapy/core/http2/stream.py
View File

@ -1,13 +1,13 @@
import logging
from enum import Enum
from io import BytesIO
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from urllib.parse import urlparse
from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
from h2.errors import ErrorCodes
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
from hpack import HeaderTuple
from twisted.internet.defer import Deferred, CancelledError
from twisted.internet.defer import CancelledError, Deferred
from twisted.internet.error import ConnectionClosed
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
@ -32,18 +32,19 @@ class InactiveStreamClosed(ConnectionClosed):
self.request = request
def __str__(self) -> str:
return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}'
return f"InactiveStreamClosed: Connection was closed without sending the request {self.request!r}"
class InvalidHostname(H2Error):
def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
def __init__(
self, request: Request, expected_hostname: str, expected_netloc: str
) -> None:
self.request = request
self.expected_hostname = expected_hostname
self.expected_netloc = expected_netloc
def __str__(self) -> str:
return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
return f"InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}"
class StreamCloseReason(Enum):
@ -100,28 +101,31 @@ class Stream:
self._request: Request = request
self._protocol: "H2ClientProtocol" = protocol
self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
self._download_maxsize = self._request.meta.get(
"download_maxsize", download_maxsize
)
self._download_warnsize = self._request.meta.get(
"download_warnsize", download_warnsize
)
# Metadata of an HTTP/2 connection stream
# initialized when stream is instantiated
self.metadata: Dict = {
'request_content_length': 0 if self._request.body is None else len(self._request.body),
"request_content_length": 0
if self._request.body is None
else len(self._request.body),
# Flag to keep track whether the stream has initiated the request
'request_sent': False,
"request_sent": False,
# Flag to track whether we have logged about exceeding download warnsize
'reached_warnsize': False,
"reached_warnsize": False,
# Each time we send a data frame, we will decrease value by the amount send.
'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
"remaining_content_length": 0
if self._request.body is None
else len(self._request.body),
# Flag to keep track whether client (self) have closed this stream
'stream_closed_local': False,
"stream_closed_local": False,
# Flag to keep track whether the server has closed the stream
'stream_closed_server': False,
"stream_closed_server": False,
}
# Private variable used to build the response
@ -130,31 +134,27 @@ class Stream:
self._response: Dict = {
# Data received frame by frame from the server is appended
# and passed to the response Deferred when completely received.
'body': BytesIO(),
"body": BytesIO(),
# The amount of data received that counts against the
# flow control window
'flow_controlled_size': 0,
"flow_controlled_size": 0,
# Headers received after sending the request
'headers': Headers({}),
"headers": Headers({}),
}
def _cancel(_) -> None:
# Close this stream as gracefully as possible
# If the associated request is initiated we reset this stream
# else we directly call close() method
if self.metadata['request_sent']:
if self.metadata["request_sent"]:
self.reset_stream(StreamCloseReason.CANCELLED)
else:
self.close(StreamCloseReason.CANCELLED)
self._deferred_response = Deferred(_cancel)
def __str__(self) -> str:
return f'Stream(id={self.stream_id!r})'
__repr__ = __str__
def __repr__(self) -> str:
return f"Stream(id={self.stream_id!r})"
@property
def _log_warnsize(self) -> bool:
@ -165,14 +165,16 @@ class Stream:
True if both the above conditions hold true
False if any of the conditions is false
"""
content_length_header = int(self._response['headers'].get(b'Content-Length', -1))
content_length_header = int(
self._response["headers"].get(b"Content-Length", -1)
)
return (
self._download_warnsize
and (
self._response['flow_controlled_size'] > self._download_warnsize
self._response["flow_controlled_size"] > self._download_warnsize
or content_length_header > self._download_warnsize
)
and not self.metadata['reached_warnsize']
and not self.metadata["reached_warnsize"]
)
def get_response(self) -> Deferred:
@ -185,9 +187,10 @@ class Stream:
# Make sure that we are sending the request to the correct URL
url = urlparse(self._request.url)
return (
url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8')
or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8')
or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
url.netloc == str(self._protocol.metadata["uri"].host, "utf-8")
or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8")
or url.netloc
== f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
)
def _get_request_headers(self) -> List[Tuple[str, str]]:
@ -195,7 +198,7 @@ class Stream:
path = url.path
if url.query:
path += '?' + url.query
path += "?" + url.query
# This pseudo-header field MUST NOT be empty for "http" or "https"
# URIs; "http" or "https" URIs that do not contain a path component
@ -204,40 +207,40 @@ class Stream:
# a path component; these MUST include a ":path" pseudo-header field
# with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
if not path:
path = '*' if self._request.method == 'OPTIONS' else '/'
path = "*" if self._request.method == "OPTIONS" else "/"
# Make sure pseudo-headers comes before all the other headers
headers = [
(':method', self._request.method),
(':authority', url.netloc),
(":method", self._request.method),
(":authority", url.netloc),
]
# The ":scheme" and ":path" pseudo-header fields MUST
# be omitted for CONNECT method (refer RFC 7540 - Section 8.3)
if self._request.method != 'CONNECT':
if self._request.method != "CONNECT":
headers += [
(':scheme', self._protocol.metadata['uri'].scheme),
(':path', path),
(":scheme", self._protocol.metadata["uri"].scheme),
(":path", path),
]
content_length = str(len(self._request.body))
headers.append(('Content-Length', content_length))
headers.append(("Content-Length", content_length))
content_length_name = self._request.headers.normkey(b'Content-Length')
content_length_name = self._request.headers.normkey(b"Content-Length")
for name, values in self._request.headers.items():
for value in values:
value = str(value, 'utf-8')
value = str(value, "utf-8")
if name == content_length_name:
if value != content_length:
logger.warning(
'Ignoring bad Content-Length header %r of request %r, '
'sending %r instead',
"Ignoring bad Content-Length header %r of request %r, "
"sending %r instead",
value,
self._request,
content_length,
)
continue
headers.append((str(name, 'utf-8'), value))
headers.append((str(name, "utf-8"), value))
return headers
@ -245,7 +248,7 @@ class Stream:
if self.check_request_url():
headers = self._get_request_headers()
self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
self.metadata['request_sent'] = True
self.metadata["request_sent"] = True
self.send_data()
else:
# Close this stream calling the response errback
@ -254,44 +257,53 @@ class Stream:
def send_data(self) -> None:
"""Called immediately after the headers are sent. Here we send all the
data as part of the request.
data as part of the request.
If the content length is 0 initially then we end the stream immediately and
wait for response data.
If the content length is 0 initially then we end the stream immediately and
wait for response data.
Warning: Only call this method when stream not closed from client side
and has initiated request already by sending HEADER frame. If not then
stream will raise ProtocolError (raise by h2 state machine).
"""
if self.metadata['stream_closed_local']:
Warning: Only call this method when stream not closed from client side
and has initiated request already by sending HEADER frame. If not then
stream will raise ProtocolError (raise by h2 state machine).
"""
if self.metadata["stream_closed_local"]:
raise StreamClosedError(self.stream_id)
# Firstly, check what the flow control window is for current stream.
window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id)
window_size = self._protocol.conn.local_flow_control_window(
stream_id=self.stream_id
)
# Next, check what the maximum frame size is.
max_frame_size = self._protocol.conn.max_outbound_frame_size
# We will send no more than the window size or the remaining file size
# of data in this call, whichever is smaller.
bytes_to_send_size = min(window_size, self.metadata['remaining_content_length'])
bytes_to_send_size = min(window_size, self.metadata["remaining_content_length"])
# We now need to send a number of data frames.
while bytes_to_send_size > 0:
chunk_size = min(bytes_to_send_size, max_frame_size)
data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length']
data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
data_chunk_start_id = (
self.metadata["request_content_length"]
- self.metadata["remaining_content_length"]
)
data_chunk = self._request.body[
data_chunk_start_id : data_chunk_start_id + chunk_size
]
self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
bytes_to_send_size -= chunk_size
self.metadata['remaining_content_length'] -= chunk_size
self.metadata["remaining_content_length"] -= chunk_size
self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
self.metadata["remaining_content_length"] = max(
0, self.metadata["remaining_content_length"]
)
# End the stream if no more data needs to be send
if self.metadata['remaining_content_length'] == 0:
if self.metadata["remaining_content_length"] == 0:
self._protocol.conn.end_stream(self.stream_id)
# Q. What about the rest of the data?
@ -303,62 +315,64 @@ class Stream:
blocked behind the flow control.
"""
if (
self.metadata['remaining_content_length']
and not self.metadata['stream_closed_server']
and self.metadata['request_sent']
self.metadata["remaining_content_length"]
and not self.metadata["stream_closed_server"]
and self.metadata["request_sent"]
):
self.send_data()
def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
self._response['body'].write(data)
self._response['flow_controlled_size'] += flow_controlled_length
self._response["body"].write(data)
self._response["flow_controlled_size"] += flow_controlled_length
# We check maxsize here in case the Content-Length header was not received
if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
if (
self._download_maxsize
and self._response["flow_controlled_size"] > self._download_maxsize
):
self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
return
if self._log_warnsize:
self.metadata['reached_warnsize'] = True
self.metadata["reached_warnsize"] = True
warning_msg = (
f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
f'warn size ({self._download_warnsize}) in request {self._request}'
f"warn size ({self._download_warnsize}) in request {self._request}"
)
logger.warning(warning_msg)
# Acknowledge the data received
self._protocol.conn.acknowledge_received_data(
self._response['flow_controlled_size'],
self.stream_id
self._response["flow_controlled_size"], self.stream_id
)
def receive_headers(self, headers: List[HeaderTuple]) -> None:
for name, value in headers:
self._response['headers'][name] = value
self._response["headers"].appendlist(name, value)
# Check if we exceed the allowed max data size which can be received
expected_size = int(self._response['headers'].get(b'Content-Length', -1))
expected_size = int(self._response["headers"].get(b"Content-Length", -1))
if self._download_maxsize and expected_size > self._download_maxsize:
self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
return
if self._log_warnsize:
self.metadata['reached_warnsize'] = True
self.metadata["reached_warnsize"] = True
warning_msg = (
f'Expected response size ({expected_size}) larger than '
f'download warn size ({self._download_warnsize}) in request {self._request}'
f"Expected response size ({expected_size}) larger than "
f"download warn size ({self._download_warnsize}) in request {self._request}"
)
logger.warning(warning_msg)
def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
"""Close this stream by sending a RST_FRAME to the remote peer"""
if self.metadata['stream_closed_local']:
if self.metadata["stream_closed_local"]:
raise StreamClosedError(self.stream_id)
# Clear buffer earlier to avoid keeping data in memory for a long time
self._response['body'].truncate(0)
self._response["body"].truncate(0)
self.metadata['stream_closed_local'] = True
self.metadata["stream_closed_local"] = True
self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
self.close(reason)
@ -368,13 +382,14 @@ class Stream:
errors: Optional[List[BaseException]] = None,
from_protocol: bool = False,
) -> None:
"""Based on the reason sent we will handle each case.
"""
if self.metadata['stream_closed_server']:
"""Based on the reason sent we will handle each case."""
if self.metadata["stream_closed_server"]:
raise StreamClosedError(self.stream_id)
if not isinstance(reason, StreamCloseReason):
raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
raise TypeError(
f"Expected StreamCloseReason, received {reason.__class__.__qualname__}"
)
# Have default value of errors as an empty list as
# some cases can add a list of exceptions
@ -383,7 +398,7 @@ class Stream:
if not from_protocol:
self._protocol.pop_stream(self.stream_id)
self.metadata['stream_closed_server'] = True
self.metadata["stream_closed_server"] = True
# We do not check for Content-Length or Transfer-Encoding in response headers
# and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
@ -394,13 +409,14 @@ class Stream:
# receiving DATA_FRAME's when we have received the headers (not
# having Content-Length)
if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
expected_size = int(self._response['headers'].get(
b'Content-Length',
self._response['flow_controlled_size'])
expected_size = int(
self._response["headers"].get(
b"Content-Length", self._response["flow_controlled_size"]
)
)
error_msg = (
f'Cancelling download of {self._request.url}: received response '
f'size ({expected_size}) larger than download max size ({self._download_maxsize})'
f"Cancelling download of {self._request.url}: received response "
f"size ({expected_size}) larger than download max size ({self._download_maxsize})"
)
logger.error(error_msg)
self._deferred_response.errback(CancelledError(error_msg))
@ -418,16 +434,20 @@ class Stream:
# There maybe no :status in headers, we make
# HTTP Status Code: 499 - Client Closed Request
self._response['headers'][':status'] = '499'
self._response["headers"][":status"] = "499"
self._fire_response_deferred()
elif reason is StreamCloseReason.RESET:
self._deferred_response.errback(ResponseFailed([
Failure(
f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
ProtocolError
self._deferred_response.errback(
ResponseFailed(
[
Failure(
f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
ProtocolError,
)
]
)
]))
)
elif reason is StreamCloseReason.CONNECTION_LOST:
self._deferred_response.errback(ResponseFailed(errors))
@ -438,33 +458,35 @@ class Stream:
else:
assert reason is StreamCloseReason.INVALID_HOSTNAME
self._deferred_response.errback(InvalidHostname(
self._request,
str(self._protocol.metadata['uri'].host, 'utf-8'),
f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
))
self._deferred_response.errback(
InvalidHostname(
self._request,
str(self._protocol.metadata["uri"].host, "utf-8"),
f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}',
)
)
def _fire_response_deferred(self) -> None:
"""Builds response from the self._response dict
and fires the response deferred callback with the
generated response instance"""
body = self._response['body'].getvalue()
body = self._response["body"].getvalue()
response_cls = responsetypes.from_args(
headers=self._response['headers'],
headers=self._response["headers"],
url=self._request.url,
body=body,
)
response = response_cls(
url=self._request.url,
status=int(self._response['headers'][':status']),
headers=self._response['headers'],
status=int(self._response["headers"][":status"]),
headers=self._response["headers"],
body=body,
request=self._request,
certificate=self._protocol.metadata['certificate'],
ip_address=self._protocol.metadata['ip_address'],
protocol='h2',
certificate=self._protocol.metadata["certificate"],
ip_address=self._protocol.metadata["ip_address"],
protocol="h2",
)
self._deferred_response.callback(response)

110
scrapy/core/scheduler.py
View File

@ -1,8 +1,7 @@
import json
import logging
import os
from abc import abstractmethod
from os.path import exists, join
from pathlib import Path
from typing import Optional, Type, TypeVar
from twisted.internet.defer import Deferred
@ -13,7 +12,6 @@ from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
logger = logging.getLogger(__name__)
@ -21,14 +19,18 @@ class BaseSchedulerMeta(type):
"""
Metaclass to check scheduler classes against the necessary interface
"""
def __instancecheck__(cls, instance):
return cls.__subclasscheck__(type(instance))
def __subclasscheck__(cls, subclass):
return (
hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests)
and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request)
and hasattr(subclass, "next_request") and callable(subclass.next_request)
hasattr(subclass, "has_pending_requests")
and callable(subclass.has_pending_requests)
and hasattr(subclass, "enqueue_request")
and callable(subclass.enqueue_request)
and hasattr(subclass, "next_request")
and callable(subclass.next_request)
)
@ -163,6 +165,7 @@ class Scheduler(BaseScheduler):
:param crawler: The crawler object corresponding to the current crawl.
:type crawler: :class:`scrapy.crawler.Crawler`
"""
def __init__(
self,
dupefilter,
@ -188,15 +191,15 @@ class Scheduler(BaseScheduler):
"""
Factory method, initializes the scheduler with arguments taken from the crawl settings
"""
dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS'])
dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"])
return cls(
dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
jobdir=job_dir(crawler.settings),
dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']),
mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']),
logunser=crawler.settings.getbool('SCHEDULER_DEBUG'),
dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]),
mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]),
logunser=crawler.settings.getbool("SCHEDULER_DEBUG"),
stats=crawler.stats,
pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']),
pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]),
crawler=crawler,
)
@ -240,11 +243,11 @@ class Scheduler(BaseScheduler):
return False
dqok = self._dqpush(request)
if dqok:
self.stats.inc_value('scheduler/enqueued/disk', spider=self.spider)
self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
else:
self._mqpush(request)
self.stats.inc_value('scheduler/enqueued/memory', spider=self.spider)
self.stats.inc_value('scheduler/enqueued', spider=self.spider)
self.stats.inc_value("scheduler/enqueued/memory", spider=self.spider)
self.stats.inc_value("scheduler/enqueued", spider=self.spider)
return True
def next_request(self) -> Optional[Request]:
@ -258,13 +261,13 @@ class Scheduler(BaseScheduler):
"""
request = self.mqs.pop()
if request is not None:
self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
else:
request = self._dqpop()
if request is not None:
self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider)
self.stats.inc_value("scheduler/dequeued/disk", spider=self.spider)
if request is not None:
self.stats.inc_value('scheduler/dequeued', spider=self.spider)
self.stats.inc_value("scheduler/dequeued", spider=self.spider)
return request
def __len__(self) -> int:
@ -280,13 +283,19 @@ class Scheduler(BaseScheduler):
self.dqs.push(request)
except ValueError as e: # non serializable request
if self.logunser:
msg = ("Unable to serialize request: %(request)s - reason:"
" %(reason)s - no more unserializable requests will be"
" logged (stats being collected)")
logger.warning(msg, {'request': request, 'reason': e},
exc_info=True, extra={'spider': self.spider})
msg = (
"Unable to serialize request: %(request)s - reason:"
" %(reason)s - no more unserializable requests will be"
" logged (stats being collected)"
)
logger.warning(
msg,
{"request": request, "reason": e},
exc_info=True,
extra={"spider": self.spider},
)
self.logunser = False
self.stats.inc_value('scheduler/unserializable', spider=self.spider)
self.stats.inc_value("scheduler/unserializable", spider=self.spider)
return False
else:
return True
@ -300,43 +309,50 @@ class Scheduler(BaseScheduler):
return None
def _mq(self):
""" Create a new priority queue instance, with in-memory storage """
return create_instance(self.pqclass,
settings=None,
crawler=self.crawler,
downstream_queue_cls=self.mqclass,
key='')
"""Create a new priority queue instance, with in-memory storage"""
return create_instance(
self.pqclass,
settings=None,
crawler=self.crawler,
downstream_queue_cls=self.mqclass,
key="",
)
def _dq(self):
""" Create a new priority queue instance, with disk storage """
"""Create a new priority queue instance, with disk storage"""
state = self._read_dqs_state(self.dqdir)
q = create_instance(self.pqclass,
settings=None,
crawler=self.crawler,
downstream_queue_cls=self.dqclass,
key=self.dqdir,
startprios=state)
q = create_instance(
self.pqclass,
settings=None,
crawler=self.crawler,
downstream_queue_cls=self.dqclass,
key=self.dqdir,
startprios=state,
)
if q:
logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
{'queuesize': len(q)}, extra={'spider': self.spider})
logger.info(
"Resuming crawl (%(queuesize)d requests scheduled)",
{"queuesize": len(q)},
extra={"spider": self.spider},
)
return q
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """
"""Return a folder name to keep disk queue state at"""
if jobdir is not None:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
dqdir = Path(jobdir, "requests.queue")
if not dqdir.exists():
dqdir.mkdir(parents=True)
return str(dqdir)
return None
def _read_dqs_state(self, dqdir: str) -> list:
path = join(dqdir, 'active.json')
if not exists(path):
path = Path(dqdir, "active.json")
if not path.exists():
return []
with open(path) as f:
with path.open(encoding="utf-8") as f:
return json.load(f)
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with open(join(dqdir, 'active.json'), 'w') as f:
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
json.dump(state, f)

222
scrapy/core/scraper.py
View File

@ -1,14 +1,28 @@
"""This module implements the Scraper component which parses responses and
extracts information from them"""
from __future__ import annotations
import logging
from collections import deque
from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
AsyncIterable,
Deque,
Generator,
Iterable,
Optional,
Set,
Tuple,
Union,
)
from itemadapter import is_item
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
from scrapy import signals, Spider
from scrapy import Spider, signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
@ -20,11 +34,13 @@ from scrapy.utils.defer import (
parallel,
parallel_async,
)
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from scrapy.crawler import Crawler
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
@ -45,7 +61,9 @@ class Slot:
self.itemproc_size: int = 0
self.closing: Optional[Deferred] = None
def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:
def add_response_request(
self, result: Union[Response, Failure], request: Request
) -> Deferred:
deferred = Deferred()
self.queue.append((result, request, deferred))
if isinstance(result, Response):
@ -59,7 +77,9 @@ class Slot:
self.active.add(request)
return response, request, deferred
def finish_response(self, result: Union[Response, Failure], request: Request) -> None:
def finish_response(
self, result: Union[Response, Failure], request: Request
) -> None:
self.active.remove(request)
if isinstance(result, Response):
self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)
@ -74,13 +94,12 @@ class Slot:
class Scraper:
def __init__(self, crawler):
def __init__(self, crawler: Crawler) -> None:
self.slot: Optional[Slot] = None
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"])
self.itemproc = itemproc_cls.from_crawler(crawler)
self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS')
self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS")
self.crawler = crawler
self.signals = crawler.signals
self.logformatter = crawler.logformatter
@ -88,7 +107,7 @@ class Scraper:
@inlineCallbacks
def open_spider(self, spider: Spider):
"""Open the given spider for scraping and allocate resources for it"""
self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
yield self.itemproc.open_spider(spider)
def close_spider(self, spider: Spider) -> Deferred:
@ -109,7 +128,9 @@ class Scraper:
if self.slot.closing and self.slot.is_idle():
self.slot.closing.callback(spider)
def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
def enqueue_scrape(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
if self.slot is None:
raise RuntimeError("Scraper slot not assigned")
dfd = self.slot.add_response_request(result, request)
@ -122,10 +143,13 @@ class Scraper:
dfd.addBoth(finish_scraping)
dfd.addErrback(
lambda f: logger.error('Scraper bug processing %(request)s',
{'request': request},
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
lambda f: logger.error(
"Scraper bug processing %(request)s",
{"request": request},
exc_info=failure_to_exc_info(f),
extra={"spider": spider},
)
)
self._scrape_next(spider)
return dfd
@ -135,35 +159,49 @@ class Scraper:
response, request, deferred = self.slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
def _scrape(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
"""
Handle the downloaded response or failure through the spider callback/errback
"""
if not isinstance(result, (Response, Failure)):
raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")
dfd = self._scrape2(result, request, spider) # returns spider's processed output
raise TypeError(
f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}"
)
dfd = self._scrape2(
result, request, spider
) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, result, spider)
dfd.addCallback(self.handle_spider_output, request, result, spider)
return dfd
def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
def _scrape2(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
"""
Handle the different cases of request's result been a Response or a Failure
"""
if isinstance(result, Response):
return self.spidermw.scrape_response(self.call_spider, result, request, spider)
else: # result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
return self.spidermw.scrape_response(
self.call_spider, result, request, spider
)
# else result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
def call_spider(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
dfd.addCallbacks(
callback=callback, callbackKeywords=result.request.cb_kwargs
)
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)
@ -171,49 +209,75 @@ class Scraper:
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
def handle_spider_error(
self, _failure: Failure, request: Request, response: Response, spider: Spider
) -> None:
exc = _failure.value
if isinstance(exc, CloseSpider):
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
assert self.crawler.engine is not None # typing
self.crawler.engine.close_spider(spider, exc.reason or "cancelled")
return
logkws = self.logformatter.spider_error(_failure, request, response, spider)
logger.log(
*logformatter_adapter(logkws),
exc_info=failure_to_exc_info(_failure),
extra={'spider': spider}
extra={"spider": spider},
)
self.signals.send_catch_log(
signal=signals.spider_error,
failure=_failure, response=response,
spider=spider
failure=_failure,
response=response,
spider=spider,
)
self.crawler.stats.inc_value(
f"spider_exceptions/{_failure.value.__class__.__name__}",
spider=spider
f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider
)
def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
response: Response, spider: Spider) -> Deferred:
def handle_spider_output(
self,
result: Union[Iterable, AsyncIterable],
request: Request,
response: Response,
spider: Spider,
) -> Deferred:
if not result:
return defer_succeed(None)
it: Union[Generator, AsyncGenerator]
if isinstance(result, AsyncIterable):
it = aiter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
it = aiter_errback(
result, self.handle_spider_error, request, response, spider
)
dfd = parallel_async(
it,
self.concurrent_items,
self._process_spidermw_output,
request,
response,
spider,
)
else:
it = iter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
it = iter_errback(
result, self.handle_spider_error, request, response, spider
)
dfd = parallel(
it,
self.concurrent_items,
self._process_spidermw_output,
request,
response,
spider,
)
return dfd
def _process_spidermw_output(self, output: Any, request: Request, response: Response,
spider: Spider) -> Optional[Deferred]:
def _process_spidermw_output(
self, output: Any, request: Request, response: Response, spider: Spider
) -> Optional[Deferred]:
"""Process each Request/Item (given in the output parameter) returned
from the given spider
"""
assert self.slot is not None # typing
if isinstance(output, Request):
assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
elif is_item(output):
self.slot.itemproc_size += 1
@ -225,14 +289,19 @@ class Scraper:
else:
typename = type(output).__name__
logger.error(
'Spider must return request, item, or None, got %(typename)r in %(request)s',
{'request': request, 'typename': typename},
extra={'spider': spider},
"Spider must return request, item, or None, got %(typename)r in %(request)s",
{"request": request, "typename": typename},
extra={"spider": spider},
)
return None
def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request,
spider: Spider) -> Union[Failure, None]:
def _log_download_errors(
self,
spider_failure: Failure,
download_failure: Failure,
request: Request,
spider: Spider,
) -> Union[Failure, None]:
"""Log and silence errors that come from the engine (typically download
errors that got propagated thru here).
@ -242,29 +311,33 @@ class Scraper:
"""
if not download_failure.check(IgnoreRequest):
if download_failure.frames:
logkws = self.logformatter.download_error(download_failure, request, spider)
logkws = self.logformatter.download_error(
download_failure, request, spider
)
logger.log(
*logformatter_adapter(logkws),
extra={'spider': spider},
extra={"spider": spider},
exc_info=failure_to_exc_info(download_failure),
)
else:
errmsg = download_failure.getErrorMessage()
if errmsg:
logkws = self.logformatter.download_error(
download_failure, request, spider, errmsg)
download_failure, request, spider, errmsg
)
logger.log(
*logformatter_adapter(logkws),
extra={'spider': spider},
extra={"spider": spider},
)
if spider_failure is not download_failure:
return spider_failure
return None
def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None:
"""ItemProcessor finished for the given ``item`` and returned ``output``
"""
def _itemproc_finished(
self, output: Any, item: Any, response: Response, spider: Spider
) -> None:
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
assert self.slot is not None # typing
self.slot.itemproc_size -= 1
if isinstance(output, Failure):
@ -272,21 +345,30 @@ class Scraper:
if isinstance(ex, DropItem):
logkws = self.logformatter.dropped(item, ex, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
else:
logkws = self.logformatter.scraped(output, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
signal=signals.item_dropped,
item=item,
response=response,
spider=spider,
exception=output.value,
)
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(
*logformatter_adapter(logkws),
extra={"spider": spider},
exc_info=failure_to_exc_info(output),
)
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)
signal=signals.item_error,
item=item,
response=response,
spider=spider,
failure=output,
)
logkws = self.logformatter.scraped(output, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response, spider=spider
)

211
scrapy/core/spidermw.py
View File

@ -6,7 +6,17 @@ See documentation in docs/topics/spider-middleware.rst
import logging
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
from typing import (
Any,
AsyncGenerator,
AsyncIterable,
Callable,
Generator,
Iterable,
Tuple,
Union,
cast,
)
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
@ -17,10 +27,14 @@ from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
maybe_deferred_to_future,
mustbe_deferred,
)
from scrapy.utils.python import MutableAsyncChain, MutableChain
logger = logging.getLogger(__name__)
@ -33,7 +47,7 @@ def _isiterable(o) -> bool:
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
component_name = "spider middleware"
def __init__(self, *middlewares):
super().__init__(*middlewares)
@ -41,28 +55,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
def _add_middleware(self, mw):
super()._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].appendleft(mw.process_start_requests)
process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
self.methods['process_spider_output'].appendleft(process_spider_output)
process_spider_exception = getattr(mw, 'process_spider_exception', None)
self.methods['process_spider_exception'].appendleft(process_spider_exception)
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
if hasattr(mw, "process_start_requests"):
self.methods["process_start_requests"].appendleft(mw.process_start_requests)
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
self.methods["process_spider_output"].appendleft(process_spider_output)
process_spider_exception = getattr(mw, "process_spider_exception", None)
self.methods["process_spider_exception"].appendleft(process_spider_exception)
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Any:
for method in self.methods['process_spider_input']:
def _process_spider_input(
self,
scrape_func: ScrapeFunc,
response: Response,
request: Request,
spider: Spider,
) -> Any:
for method in self.methods["process_spider_input"]:
method = cast(Callable, method)
try:
result = method(response=response, spider=spider)
if result is not None:
msg = (f"{method.__qualname__} must return None "
f"or raise an exception, got {type(result)}")
msg = (
f"{method.__qualname__} must return None "
f"or raise an exception, got {type(result)}"
)
raise _InvalidOutput(msg)
except _InvalidOutput:
raise
@ -70,17 +91,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
) -> Union[Generator, AsyncGenerator]:
def _evaluate_iterable(
self,
response: Response,
spider: Spider,
iterable: Union[Iterable, AsyncIterable],
exception_processor_index: int,
recover_to: Union[MutableChain, MutableAsyncChain],
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable):
try:
for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(response, spider, Failure(ex),
exception_processor_index)
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
)
if isinstance(exception_result, Failure):
raise
recover_to.extend(exception_result)
@ -90,8 +116,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
async for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(response, spider, Failure(ex),
exception_processor_index)
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
)
if isinstance(exception_result, Failure):
raise
recover_to.extend(exception_result)
@ -100,13 +127,20 @@ class SpiderMiddlewareManager(MiddlewareManager):
return process_async(iterable)
return process_sync(iterable)
def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
start_index: int = 0) -> Union[Failure, MutableChain]:
def _process_spider_exception(
self,
response: Response,
spider: Spider,
_failure: Failure,
start_index: int = 0,
) -> Union[Failure, MutableChain]:
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
return _failure
method_list = islice(self.methods['process_spider_exception'], start_index, None)
method_list = islice(
self.methods["process_spider_exception"], start_index, None
)
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
@ -115,22 +149,25 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
dfd: Deferred = self._process_spider_output(
response, spider, result, method_index + 1
)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return dfd.result
else:
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
raise _InvalidOutput(msg)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
raise _InvalidOutput(msg)
elif result is None:
continue
else:
msg = (f"{method.__qualname__} must return None "
f"or an iterable, got {type(result)}")
msg = (
f"{method.__qualname__} must return None "
f"or an iterable, got {type(result)}"
)
raise _InvalidOutput(msg)
return _failure
@ -138,9 +175,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
# being available immediately which doesn't work when it's a wrapped coroutine.
# It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
@inlineCallbacks
def _process_spider_output(self, response: Response, spider: Spider,
result: Union[Iterable, AsyncIterable], start_index: int = 0
) -> Deferred:
def _process_spider_output(
self,
response: Response,
spider: Spider,
result: Union[Iterable, AsyncIterable],
start_index: int = 0,
) -> Deferred:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered: Union[MutableChain, MutableAsyncChain]
@ -157,7 +198,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
# Storing methods and method tuples in the same list is weird but we should be able to roll this back
# when we drop this compatibility feature.
method_list = islice(self.methods['process_spider_output'], start_index, None)
method_list = islice(self.methods["process_spider_output"], start_index, None)
for method_index, method_pair in enumerate(method_list, start=start_index):
if method_pair is None:
continue
@ -178,24 +219,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
result = as_async_generator(result)
elif need_downgrade:
if not self.downgrade_warning_done:
logger.warning(f"Async iterable passed to {method.__qualname__} "
f"was downgraded to a non-async one")
logger.warning(
f"Async iterable passed to {method.__qualname__} "
f"was downgraded to a non-async one"
)
self.downgrade_warning_done = True
assert isinstance(result, AsyncIterable)
# AsyncIterable -> Iterable
result = yield deferred_from_coro(collect_asyncgen(result))
if isinstance(recovered, AsyncIterable):
recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
recovered_collected = yield deferred_from_coro(
collect_asyncgen(recovered)
)
recovered = MutableChain(recovered_collected)
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1)
exception_result = self._process_spider_exception(
response, spider, Failure(ex), method_index + 1
)
if isinstance(exception_result, Failure):
raise
return exception_result
if _isiterable(result):
result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
result = self._evaluate_iterable(
response, spider, result, method_index + 1, recovered
)
else:
if iscoroutine(result):
result.close() # Silence warning about not awaiting
@ -213,61 +262,81 @@ class SpiderMiddlewareManager(MiddlewareManager):
if last_result_is_async:
return MutableAsyncChain(result, recovered)
else:
return MutableChain(result, recovered) # type: ignore[arg-type]
return MutableChain(result, recovered) # type: ignore[arg-type]
async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
async def _process_callback_output(
self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
recovered: Union[MutableChain, MutableAsyncChain]
if isinstance(result, AsyncIterable):
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
result = self._evaluate_iterable(response, spider, result, 0, recovered)
result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
result = await maybe_deferred_to_future(
self._process_spider_output(response, spider, result)
)
if isinstance(result, AsyncIterable):
return MutableAsyncChain(result, recovered)
else:
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered) # type: ignore[arg-type]
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered) # type: ignore[arg-type]
def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Deferred:
async def process_callback_output(result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
def scrape_response(
self,
scrape_func: ScrapeFunc,
response: Response,
request: Request,
spider: Spider,
) -> Deferred:
async def process_callback_output(
result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
return await self._process_callback_output(response, spider, result)
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
dfd = mustbe_deferred(
self._process_spider_input, scrape_func, response, request, spider
)
dfd.addCallbacks(
callback=deferred_f_from_coro_f(process_callback_output),
errback=process_spider_exception,
)
return dfd
def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
return self._process_chain('process_start_requests', start_requests, spider)
return self._process_chain("process_start_requests", start_requests, spider)
# This method is only needed until _async compatibility methods are removed.
@staticmethod
def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
def _get_async_method_pair(
mw: Any, methodname: str
) -> Union[None, Callable, Tuple[Callable, Callable]]:
normal_method = getattr(mw, methodname, None)
methodname_async = methodname + "_async"
async_method = getattr(mw, methodname_async, None)
if not async_method:
return normal_method
if not normal_method:
logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
f"without {methodname}, skipping this method.")
logger.error(
f"Middleware {mw.__qualname__} has {methodname_async} "
f"without {methodname}, skipping this method."
)
return None
if not isasyncgenfunction(async_method):
logger.error(f"{async_method.__qualname__} is not "
f"an async generator function, skipping this method.")
logger.error(
f"{async_method.__qualname__} is not "
f"an async generator function, skipping this method."
)
return normal_method
if isasyncgenfunction(normal_method):
logger.error(f"{normal_method.__qualname__} is an async "
f"generator function while {methodname_async} exists, "
f"skipping both methods.")
logger.error(
f"{normal_method.__qualname__} is an async "
f"generator function while {methodname_async} exists, "
f"skipping both methods."
)
return None
return normal_method, async_method

91
scrapy/crawler.py
View File

@ -1,7 +1,10 @@
from __future__ import annotations
import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Optional
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@ -14,20 +17,20 @@ except ImportError:
from zope.interface.verify import verifyClass
from scrapy import signals, Spider
from scrapy import Spider, signals
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import overridden_settings, Settings
from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.utils.log import (
LogCounterHandler,
configure_logging,
get_scrapy_root_handler,
install_scrapy_root_handler,
log_reactor_info,
log_scrapy_info,
LogCounterHandler,
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
@ -38,15 +41,17 @@ from scrapy.utils.reactor import (
verify_installed_reactor,
)
if TYPE_CHECKING:
from scrapy.utils.request import RequestFingerprinter
logger = logging.getLogger(__name__)
class Crawler:
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
if isinstance(spidercls, Spider):
raise ValueError('The spidercls argument must be a class, not an object')
raise ValueError("The spidercls argument must be a class, not an object")
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
@ -57,14 +62,15 @@ class Crawler:
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings:\n%(settings)s",
{'settings': pprint.pformat(d)})
logger.info(
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
@ -74,11 +80,11 @@ class Crawler:
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
lf_cls = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
settings=self.settings,
crawler=self,
)
@ -103,7 +109,7 @@ class Crawler:
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
@ -154,23 +160,26 @@ class CrawlerRunner:
crawlers = property(
lambda self: self._crawlers,
doc="Set of :class:`crawlers <scrapy.crawler.Crawler>` started by "
":meth:`crawl` and managed by this class."
":meth:`crawl` and managed by this class.",
)
@staticmethod
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
cls_path = settings.get('SPIDER_LOADER_CLASS')
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
excs = (
(DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
)
try:
verifyClass(ISpiderLoader, loader_cls)
except excs:
warnings.warn(
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
'Please add all missing methods to avoid unexpected runtime errors.',
category=ScrapyDeprecationWarning, stacklevel=2
"SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does "
"not fully implement scrapy.interfaces.ISpiderLoader interface. "
"Please add all missing methods to avoid unexpected runtime errors.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return loader_cls.from_settings(settings.frozencopy())
@ -185,9 +194,12 @@ class CrawlerRunner:
@property
def spiders(self):
warnings.warn("CrawlerRunner.spiders attribute is renamed to "
"CrawlerRunner.spider_loader.",
category=ScrapyDeprecationWarning, stacklevel=2)
warnings.warn(
"CrawlerRunner.spiders attribute is renamed to "
"CrawlerRunner.spider_loader.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return self.spider_loader
def crawl(self, crawler_or_spidercls, *args, **kwargs):
@ -214,8 +226,9 @@ class CrawlerRunner:
"""
if isinstance(crawler_or_spidercls, Spider):
raise ValueError(
'The crawler_or_spidercls argument cannot be a spider object, '
'it must be a spider class (or a Crawler object)')
"The crawler_or_spidercls argument cannot be a spider object, "
"it must be a spider class (or a Crawler object)"
)
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
@ -227,7 +240,7 @@ class CrawlerRunner:
def _done(result):
self.crawlers.discard(crawler)
self._active.discard(d)
self.bootstrap_failed |= not getattr(crawler, 'spider', None)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
return result
return d.addBoth(_done)
@ -245,8 +258,9 @@ class CrawlerRunner:
"""
if isinstance(crawler_or_spidercls, Spider):
raise ValueError(
'The crawler_or_spidercls argument cannot be a spider object, '
'it must be a spider class (or a Crawler object)')
"The crawler_or_spidercls argument cannot be a spider object, "
"it must be a spider class (or a Crawler object)"
)
if isinstance(crawler_or_spidercls, Crawler):
return crawler_or_spidercls
return self._create_crawler(crawler_or_spidercls)
@ -308,18 +322,23 @@ class CrawlerProcess(CrawlerRunner):
def _signal_shutdown(self, signum, _):
from twisted.internet import reactor
install_shutdown_handlers(self._signal_kill)
signame = signal_names[signum]
logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
{'signame': signame})
logger.info(
"Received %(signame)s, shutting down gracefully. Send again to force ",
{"signame": signame},
)
reactor.callFromThread(self._graceful_stop_reactor)
def _signal_kill(self, signum, _):
from twisted.internet import reactor
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
logger.info('Received %(signame)s twice, forcing unclean shutdown',
{'signame': signame})
logger.info(
"Received %(signame)s twice, forcing unclean shutdown", {"signame": signame}
)
reactor.callFromThread(self._stop_reactor)
def _create_crawler(self, spidercls):
@ -345,6 +364,7 @@ class CrawlerProcess(CrawlerRunner):
handlers (default: True)
"""
from twisted.internet import reactor
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
@ -358,8 +378,8 @@ class CrawlerProcess(CrawlerRunner):
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def _graceful_stop_reactor(self):
@ -369,6 +389,7 @@ class CrawlerProcess(CrawlerRunner):
def _stop_reactor(self, _=None):
from twisted.internet import reactor
try:
reactor.stop()
except RuntimeError: # raised if already stopped or in shutdown stage

35
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,12 +1,11 @@
import re
import logging
import re
from w3lib import html
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse
logger = logging.getLogger(__name__)
@ -17,14 +16,14 @@ class AjaxCrawlMiddleware:
"""
def __init__(self, settings):
if not settings.getbool('AJAXCRAWL_ENABLED'):
if not settings.getbool("AJAXCRAWL_ENABLED"):
raise NotConfigured
# XXX: Google parses at least first 100k bytes; scrapy's redirect
# middleware parses first 4k. 4k turns out to be insufficient
# for this middleware, and parsing 100k could be slow.
# We use something in between (32K) by default.
self.lookup_bytes = settings.getint('AJAXCRAWL_MAXSIZE', 32768)
self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
@classmethod
def from_crawler(cls, crawler):
@ -35,23 +34,25 @@ class AjaxCrawlMiddleware:
if not isinstance(response, HtmlResponse) or response.status != 200:
return response
if request.method != 'GET':
if request.method != "GET":
# other HTTP methods are either not safe or don't have a body
return response
if 'ajax_crawlable' in request.meta: # prevent loops
if "ajax_crawlable" in request.meta: # prevent loops
return response
if not self._has_ajax_crawlable_variant(response):
return response
# scrapy already handles #! links properly
ajax_crawl_request = request.replace(url=request.url + '#!')
logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
{'ajax_crawl_request': ajax_crawl_request, 'request': request},
extra={'spider': spider})
ajax_crawl_request = request.replace(url=request.url + "#!")
logger.debug(
"Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
{"ajax_crawl_request": ajax_crawl_request, "request": request},
extra={"spider": spider},
)
ajax_crawl_request.meta['ajax_crawlable'] = True
ajax_crawl_request.meta["ajax_crawlable"] = True
return ajax_crawl_request
def _has_ajax_crawlable_variant(self, response):
@ -59,12 +60,14 @@ class AjaxCrawlMiddleware:
Return True if a page without hash fragment could be "AJAX crawlable"
according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
"""
body = response.text[:self.lookup_bytes]
body = response.text[: self.lookup_bytes]
return _has_ajaxcrawlable_meta(body)
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
_ajax_crawlable_re = re.compile(
r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'
)
def _has_ajaxcrawlable_meta(text):
@ -82,12 +85,12 @@ def _has_ajaxcrawlable_meta(text):
# Stripping scripts and comments is slow (about 20x slower than
# just checking if a string is in text); this is a quick fail-fast
# path that should work for most pages.
if 'fragment' not in text:
if "fragment" not in text:
return False
if 'content' not in text:
if "content" not in text:
return False
text = html.remove_tags_with_content(text, ('script', 'noscript'))
text = html.remove_tags_with_content(text, ("script", "noscript"))
text = html.replace_entities(text)
text = html.remove_comments(text)
return _ajax_crawlable_re.search(text) is not None

38
scrapy/downloadermiddlewares/cookies.py
View File

@ -9,7 +9,6 @@ from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@ -30,14 +29,14 @@ class CookiesMiddleware:
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('COOKIES_ENABLED'):
if not crawler.settings.getbool("COOKIES_ENABLED"):
raise NotConfigured
return cls(crawler.settings.getbool('COOKIES_DEBUG'))
return cls(crawler.settings.getbool("COOKIES_DEBUG"))
def _process_cookies(self, cookies, *, jar, request):
for cookie in cookies:
cookie_domain = cookie.domain
if cookie_domain.startswith('.'):
if cookie_domain.startswith("."):
cookie_domain = cookie_domain[1:]
request_domain = urlparse_cached(request).hostname.lower()
@ -50,7 +49,7 @@ class CookiesMiddleware:
jar.set_cookie_if_ok(cookie, request)
def process_request(self, request, spider):
if request.meta.get('dont_merge_cookies', False):
if request.meta.get("dont_merge_cookies", False):
return
cookiejarkey = request.meta.get("cookiejar")
@ -59,12 +58,12 @@ class CookiesMiddleware:
self._process_cookies(cookies, jar=jar, request=request)
# set Cookie header
request.headers.pop('Cookie', None)
request.headers.pop("Cookie", None)
jar.add_cookie_header(request)
self._debug_cookie(request, spider)
def process_response(self, request, response, spider):
if request.meta.get('dont_merge_cookies', False):
if request.meta.get("dont_merge_cookies", False):
return response
# extract cookies from Set-Cookie and drop invalid/expired cookies
@ -79,21 +78,25 @@ class CookiesMiddleware:
def _debug_cookie(self, request, spider):
if self.debug:
cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
cl = [
to_unicode(c, errors="replace")
for c in request.headers.getlist("Cookie")
]
if cl:
cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
logger.debug(msg, extra={"spider": spider})
def _debug_set_cookie(self, response, spider):
if self.debug:
cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
cl = [
to_unicode(c, errors="replace")
for c in response.headers.getlist("Set-Cookie")
]
if cl:
cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
logger.debug(msg, extra={"spider": spider})
def _format_cookie(self, cookie, request):
"""
@ -114,8 +117,11 @@ class CookiesMiddleware:
try:
decoded[key] = cookie[key].decode("utf8")
except UnicodeDecodeError:
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
request, cookie)
logger.warning(
"Non UTF-8 encoded cookie found in request %s: %s",
request,
cookie,
)
decoded[key] = cookie[key].decode("latin1", errors="replace")
cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
@ -129,7 +135,7 @@ class CookiesMiddleware:
"""
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies

22
scrapy/downloadermiddlewares/decompression.py
View File

@ -14,9 +14,8 @@ from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
warn(
'scrapy.downloadermiddlewares.decompression is deprecated',
"scrapy.downloadermiddlewares.decompression is deprecated",
ScrapyDeprecationWarning,
stacklevel=2,
)
@ -26,15 +25,15 @@ logger = logging.getLogger(__name__)
class DecompressionMiddleware:
""" This middleware tries to recognise and extract the possibly compressed
responses that may arrive. """
"""This middleware tries to recognise and extract the possibly compressed
responses that may arrive."""
def __init__(self):
self._formats = {
'tar': self._is_tar,
'zip': self._is_zip,
'gz': self._is_gzip,
'bz2': self._is_bzip2
"tar": self._is_tar,
"zip": self._is_zip,
"gz": self._is_gzip,
"bz2": self._is_bzip2,
}
def _is_tar(self, response):
@ -86,7 +85,10 @@ class DecompressionMiddleware:
for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',
{'responsefmt': fmt}, extra={'spider': spider})
logger.debug(
"Decompressed response with format: %(responsefmt)s",
{"responsefmt": fmt},
extra={"spider": spider},
)
return new_response
return response

3
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -8,13 +8,12 @@ from scrapy.utils.python import without_none_values
class DefaultHeadersMiddleware:
def __init__(self, headers):
self._headers = headers
@classmethod
def from_crawler(cls, crawler):
headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
return cls(headers.items())
def process_request(self, request, spider):

7
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -8,19 +8,18 @@ from scrapy import signals
class DownloadTimeoutMiddleware:
def __init__(self, timeout=180):
self._timeout = timeout
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
self._timeout = getattr(spider, 'download_timeout', self._timeout)
self._timeout = getattr(spider, "download_timeout", self._timeout)
def process_request(self, request, spider):
if self._timeout:
request.meta.setdefault('download_timeout', self._timeout)
request.meta.setdefault("download_timeout", self._timeout)

24
scrapy/downloadermiddlewares/httpauth.py
View File

@ -24,27 +24,29 @@ class HttpAuthMiddleware:
return o
def spider_opened(self, spider):
usr = getattr(spider, 'http_user', '')
pwd = getattr(spider, 'http_pass', '')
usr = getattr(spider, "http_user", "")
pwd = getattr(spider, "http_pass", "")
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
if not hasattr(spider, 'http_auth_domain'):
warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
'problems if the spider makes requests to several different domains. http_auth_domain '
'will be set to the domain of the first request, please set it to the correct value '
'explicitly.',
category=ScrapyDeprecationWarning)
if not hasattr(spider, "http_auth_domain"):
warnings.warn(
"Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "
"problems if the spider makes requests to several different domains. http_auth_domain "
"will be set to the domain of the first request, please set it to the correct value "
"explicitly.",
category=ScrapyDeprecationWarning,
)
self.domain_unset = True
else:
self.domain = spider.http_auth_domain
self.domain_unset = False
def process_request(self, request, spider):
auth = getattr(self, 'auth', None)
if auth and b'Authorization' not in request.headers:
auth = getattr(self, "auth", None)
if auth and b"Authorization" not in request.headers:
domain = urlparse_cached(request).hostname
if self.domain_unset:
self.domain = domain
self.domain_unset = False
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
request.headers[b'Authorization'] = auth
request.headers[b"Authorization"] = auth

83
scrapy/downloadermiddlewares/httpcache.py
View File

@ -23,27 +23,36 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
ConnectionRefusedError, ConnectionDone, ConnectError,
ConnectionLost, TCPTimedOutError, ResponseFailed,
IOError)
DOWNLOAD_EXCEPTIONS = (
defer.TimeoutError,
TimeoutError,
DNSLookupError,
ConnectionRefusedError,
ConnectionDone,
ConnectError,
ConnectionLost,
TCPTimedOutError,
ResponseFailed,
IOError,
)
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
if not settings.getbool('HTTPCACHE_ENABLED'):
if not settings.getbool("HTTPCACHE_ENABLED"):
raise NotConfigured
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
self.policy = load_object(settings["HTTPCACHE_POLICY"])(settings)
self.storage = load_object(settings["HTTPCACHE_STORAGE"])(settings)
self.ignore_missing = settings.getbool("HTTPCACHE_IGNORE_MISSING")
self.stats = stats
@classmethod
def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
def from_crawler(
cls: Type[HttpCacheMiddlewareTV], crawler: Crawler
) -> HttpCacheMiddlewareTV:
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@ -56,78 +65,86 @@ class HttpCacheMiddleware:
self.storage.close_spider(spider)
def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
if request.meta.get('dont_cache', False):
if request.meta.get("dont_cache", False):
return None
# Skip uncacheable requests
if not self.policy.should_cache_request(request):
request.meta['_dont_cache'] = True # flag as uncacheable
request.meta["_dont_cache"] = True # flag as uncacheable
return None
# Look for cached response and check if expired
cachedresponse = self.storage.retrieve_response(spider, request)
if cachedresponse is None:
self.stats.inc_value('httpcache/miss', spider=spider)
self.stats.inc_value("httpcache/miss", spider=spider)
if self.ignore_missing:
self.stats.inc_value('httpcache/ignore', spider=spider)
self.stats.inc_value("httpcache/ignore", spider=spider)
raise IgnoreRequest(f"Ignored request not in cache: {request}")
return None # first time request
# Return cached response only if not expired
cachedresponse.flags.append('cached')
cachedresponse.flags.append("cached")
if self.policy.is_cached_response_fresh(cachedresponse, request):
self.stats.inc_value('httpcache/hit', spider=spider)
self.stats.inc_value("httpcache/hit", spider=spider)
return cachedresponse
# Keep a reference to cached response to avoid a second cache lookup on
# process_response hook
request.meta['cached_response'] = cachedresponse
request.meta["cached_response"] = cachedresponse
return None
def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
if request.meta.get('dont_cache', False):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Response:
if request.meta.get("dont_cache", False):
return response
# Skip cached responses and uncacheable requests
if 'cached' in response.flags or '_dont_cache' in request.meta:
request.meta.pop('_dont_cache', None)
if "cached" in response.flags or "_dont_cache" in request.meta:
request.meta.pop("_dont_cache", None)
return response
# RFC2616 requires origin server to set Date header,
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
if 'Date' not in response.headers:
response.headers['Date'] = formatdate(usegmt=True)
if "Date" not in response.headers:
response.headers["Date"] = formatdate(usegmt=True)
# Do not validate first-hand responses
cachedresponse = request.meta.pop('cached_response', None)
cachedresponse = request.meta.pop("cached_response", None)
if cachedresponse is None:
self.stats.inc_value('httpcache/firsthand', spider=spider)
self.stats.inc_value("httpcache/firsthand", spider=spider)
self._cache_response(spider, response, request, cachedresponse)
return response
if self.policy.is_cached_response_valid(cachedresponse, response, request):
self.stats.inc_value('httpcache/revalidate', spider=spider)
self.stats.inc_value("httpcache/revalidate", spider=spider)
return cachedresponse
self.stats.inc_value('httpcache/invalidate', spider=spider)
self.stats.inc_value("httpcache/invalidate", spider=spider)
self._cache_response(spider, response, request, cachedresponse)
return response
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Optional[Response]:
cachedresponse = request.meta.pop('cached_response', None)
if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
self.stats.inc_value('httpcache/errorrecovery', spider=spider)
cachedresponse = request.meta.pop("cached_response", None)
if cachedresponse is not None and isinstance(
exception, self.DOWNLOAD_EXCEPTIONS
):
self.stats.inc_value("httpcache/errorrecovery", spider=spider)
return cachedresponse
return None
def _cache_response(
self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
self,
spider: Spider,
response: Response,
request: Request,
cachedresponse: Optional[Response],
) -> None:
if self.policy.should_cache_response(response, request):
self.stats.inc_value('httpcache/store', spider=spider)
self.stats.inc_value("httpcache/store", spider=spider)
self.storage.store_response(spider, request, response)
else:
self.stats.inc_value('httpcache/uncacheable', spider=spider)
self.stats.inc_value("httpcache/uncacheable", spider=spider)

41
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -8,18 +8,19 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
try:
import brotli
ACCEPTED_ENCODINGS.append(b'br')
ACCEPTED_ENCODINGS.append(b"br")
except ImportError:
pass
try:
import zstandard
ACCEPTED_ENCODINGS.append(b'zstd')
ACCEPTED_ENCODINGS.append(b"zstd")
except ImportError:
pass
@ -27,12 +28,13 @@ except ImportError:
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
def __init__(self, stats=None):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('COMPRESSION_ENABLED'):
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
try:
return cls(stats=crawler.stats)
@ -48,21 +50,26 @@ class HttpCompressionMiddleware:
return result
def process_request(self, request, spider):
request.headers.setdefault('Accept-Encoding',
b", ".join(ACCEPTED_ENCODINGS))
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
def process_response(self, request, response, spider):
if request.method == 'HEAD':
if request.method == "HEAD":
return response
if isinstance(response, Response):
content_encoding = response.headers.getlist('Content-Encoding')
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
if self.stats:
self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
self.stats.inc_value('httpcompression/response_count', spider=spider)
self.stats.inc_value(
"httpcompression/response_bytes",
len(decoded_body),
spider=spider,
)
self.stats.inc_value(
"httpcompression/response_count", spider=spider
)
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
@ -70,18 +77,18 @@ class HttpCompressionMiddleware:
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs['encoding'] = None
kwargs["encoding"] = None
response = response.replace(**kwargs)
if not content_encoding:
del response.headers['Content-Encoding']
del response.headers["Content-Encoding"]
return response
def _decode(self, body, encoding):
if encoding == b'gzip' or encoding == b'x-gzip':
if encoding == b"gzip" or encoding == b"x-gzip":
body = gunzip(body)
if encoding == b'deflate':
if encoding == b"deflate":
try:
body = zlib.decompress(body)
except zlib.error:
@ -91,9 +98,9 @@ class HttpCompressionMiddleware:
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
body = zlib.decompress(body, -15)
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
# Using its streaming API since its simple API could handle only cases
# where there is content size data embedded in the frame
reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))

58
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -1,6 +1,6 @@
import base64
from urllib.parse import unquote, urlunparse
from urllib.request import getproxies, proxy_bypass, _parse_proxy
from urllib.request import _parse_proxy, getproxies, proxy_bypass
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
@ -8,8 +8,7 @@ from scrapy.utils.python import to_bytes
class HttpProxyMiddleware:
def __init__(self, auth_encoding='latin-1'):
def __init__(self, auth_encoding="latin-1"):
self.auth_encoding = auth_encoding
self.proxies = {}
for type_, url in getproxies().items():
@ -22,20 +21,20 @@ class HttpProxyMiddleware:
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
if not crawler.settings.getbool("HTTPPROXY_ENABLED"):
raise NotConfigured
auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
return cls(auth_encoding)
def _basic_auth_header(self, username, password):
user_pass = to_bytes(
f'{unquote(username)}:{unquote(password)}',
encoding=self.auth_encoding)
f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding
)
return base64.b64encode(user_pass)
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))
if user:
creds = self._basic_auth_header(user, password)
@ -46,39 +45,36 @@ class HttpProxyMiddleware:
def process_request(self, request, spider):
creds, proxy_url = None, None
if 'proxy' in request.meta:
if request.meta['proxy'] is not None:
creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
if "proxy" in request.meta:
if request.meta["proxy"] is not None:
creds, proxy_url = self._get_proxy(request.meta["proxy"], "")
elif self.proxies:
parsed = urlparse_cached(request)
scheme = parsed.scheme
if (
(
# 'no_proxy' is only supported by http schemes
scheme not in ('http', 'https')
or not proxy_bypass(parsed.hostname)
)
and scheme in self.proxies
):
# 'no_proxy' is only supported by http schemes
scheme not in ("http", "https")
or not proxy_bypass(parsed.hostname)
) and scheme in self.proxies:
creds, proxy_url = self.proxies[scheme]
self._set_proxy_and_creds(request, proxy_url, creds)
def _set_proxy_and_creds(self, request, proxy_url, creds):
if proxy_url:
request.meta['proxy'] = proxy_url
elif request.meta.get('proxy') is not None:
request.meta['proxy'] = None
request.meta["proxy"] = proxy_url
elif request.meta.get("proxy") is not None:
request.meta["proxy"] = None
if creds:
request.headers[b'Proxy-Authorization'] = b'Basic ' + creds
request.meta['_auth_proxy'] = proxy_url
elif '_auth_proxy' in request.meta:
if proxy_url != request.meta['_auth_proxy']:
if b'Proxy-Authorization' in request.headers:
del request.headers[b'Proxy-Authorization']
del request.meta['_auth_proxy']
elif b'Proxy-Authorization' in request.headers:
request.headers[b"Proxy-Authorization"] = b"Basic " + creds
request.meta["_auth_proxy"] = proxy_url
elif "_auth_proxy" in request.meta:
if proxy_url != request.meta["_auth_proxy"]:
if b"Proxy-Authorization" in request.headers:
del request.headers[b"Proxy-Authorization"]
del request.meta["_auth_proxy"]
elif b"Proxy-Authorization" in request.headers:
if proxy_url:
request.meta['_auth_proxy'] = proxy_url
request.meta["_auth_proxy"] = proxy_url
else:
del request.headers[b'Proxy-Authorization']
del request.headers[b"Proxy-Authorization"]

88
scrapy/downloadermiddlewares/redirect.py
View File

@ -3,11 +3,10 @@ from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
@ -18,58 +17,66 @@ def _build_redirect_request(source_request, *, url, **kwargs):
**kwargs,
cookies=None,
)
if 'Cookie' in redirect_request.headers:
if "Cookie" in redirect_request.headers:
source_request_netloc = urlparse_cached(source_request).netloc
redirect_request_netloc = urlparse_cached(redirect_request).netloc
if source_request_netloc != redirect_request_netloc:
del redirect_request.headers['Cookie']
del redirect_request.headers["Cookie"]
return redirect_request
class BaseRedirectMiddleware:
enabled_setting = 'REDIRECT_ENABLED'
enabled_setting = "REDIRECT_ENABLED"
def __init__(self, settings):
if not settings.getbool(self.enabled_setting):
raise NotConfigured
self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES")
self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST")
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings)
def _redirect(self, redirected, request, spider, reason):
ttl = request.meta.setdefault('redirect_ttl', self.max_redirect_times)
redirects = request.meta.get('redirect_times', 0) + 1
ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
redirects = request.meta.get("redirect_times", 0) + 1
if ttl and redirects <= self.max_redirect_times:
redirected.meta['redirect_times'] = redirects
redirected.meta['redirect_ttl'] = ttl - 1
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
redirected.meta["redirect_times"] = redirects
redirected.meta["redirect_ttl"] = ttl - 1
redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [
request.url
]
redirected.meta["redirect_reasons"] = request.meta.get(
"redirect_reasons", []
) + [reason]
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
{'reason': reason, 'redirected': redirected, 'request': request},
extra={'spider': spider})
logger.debug(
"Redirecting (%(reason)s) to %(redirected)s from %(request)s",
{"reason": reason, "redirected": redirected, "request": request},
extra={"spider": spider},
)
return redirected
else:
logger.debug("Discarding %(request)s: max redirections reached",
{'request': request}, extra={'spider': spider})
raise IgnoreRequest("max redirections reached")
logger.debug(
"Discarding %(request)s: max redirections reached",
{"request": request},
extra={"spider": spider},
)
raise IgnoreRequest("max redirections reached")
def _redirect_request_using_get(self, request, redirect_url):
redirect_request = _build_redirect_request(
request,
url=redirect_url,
method='GET',
body='',
method="GET",
body="",
)
redirect_request.headers.pop('Content-Type', None)
redirect_request.headers.pop('Content-Length', None)
redirect_request.headers.pop("Content-Type", None)
redirect_request.headers.pop("Content-Length", None)
return redirect_request
@ -81,25 +88,25 @@ class RedirectMiddleware(BaseRedirectMiddleware):
def process_response(self, request, response, spider):
if (
request.meta.get('dont_redirect', False)
or response.status in getattr(spider, 'handle_httpstatus_list', [])
or response.status in request.meta.get('handle_httpstatus_list', [])
or request.meta.get('handle_httpstatus_all', False)
request.meta.get("dont_redirect", False)
or response.status in getattr(spider, "handle_httpstatus_list", [])
or response.status in request.meta.get("handle_httpstatus_list", [])
or request.meta.get("handle_httpstatus_all", False)
):
return response
allowed_status = (301, 302, 303, 307, 308)
if 'Location' not in response.headers or response.status not in allowed_status:
if "Location" not in response.headers or response.status not in allowed_status:
return response
location = safe_url_string(response.headers['Location'])
if response.headers['Location'].startswith(b'//'):
location = safe_url_string(response.headers["Location"])
if response.headers["Location"].startswith(b"//"):
request_scheme = urlparse(request.url).scheme
location = request_scheme + '://' + location.lstrip('/')
location = request_scheme + "://" + location.lstrip("/")
redirected_url = urljoin(request.url, location)
if response.status in (301, 307, 308) or request.method == 'HEAD':
if response.status in (301, 307, 308) or request.method == "HEAD":
redirected = _build_redirect_request(request, url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
@ -109,25 +116,24 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = 'METAREFRESH_ENABLED'
enabled_setting = "METAREFRESH_ENABLED"
def __init__(self, settings):
super().__init__(settings)
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay = settings.getint("METAREFRESH_MAXDELAY")
def process_response(self, request, response, spider):
if (
request.meta.get('dont_redirect', False)
or request.method == 'HEAD'
request.meta.get("dont_redirect", False)
or request.method == "HEAD"
or not isinstance(response, HtmlResponse)
):
return response
interval, url = get_meta_refresh(response,
ignore_tags=self._ignore_tags)
interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
if url and interval < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, 'meta refresh')
return self._redirect(redirected, request, spider, "meta refresh")
return response

79
scrapy/downloadermiddlewares/retry.py
View File

@ -9,7 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
from logging import getLogger, Logger
from logging import Logger, getLogger
from typing import Optional, Union
from twisted.internet import defer
@ -31,7 +31,6 @@ from scrapy.spiders import Spider
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
retry_logger = getLogger(__name__)
@ -39,11 +38,11 @@ def get_retry_request(
request: Request,
*,
spider: Spider,
reason: Union[str, Exception] = 'unspecified',
reason: Union[str, Exception] = "unspecified",
max_retry_times: Optional[int] = None,
priority_adjust: Optional[int] = None,
logger: Logger = retry_logger,
stats_base_key: str = 'retry',
stats_base_key: str = "retry",
):
"""
Returns a new :class:`~scrapy.Request` object to retry the specified
@ -87,22 +86,22 @@ def get_retry_request(
"""
settings = spider.crawler.settings
stats = spider.crawler.stats
retry_times = request.meta.get('retry_times', 0) + 1
retry_times = request.meta.get("retry_times", 0) + 1
if max_retry_times is None:
max_retry_times = request.meta.get('max_retry_times')
max_retry_times = request.meta.get("max_retry_times")
if max_retry_times is None:
max_retry_times = settings.getint('RETRY_TIMES')
max_retry_times = settings.getint("RETRY_TIMES")
if retry_times <= max_retry_times:
logger.debug(
"Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider}
{"request": request, "retry_times": retry_times, "reason": reason},
extra={"spider": spider},
)
new_request: Request = request.copy()
new_request.meta['retry_times'] = retry_times
new_request.meta["retry_times"] = retry_times
new_request.dont_filter = True
if priority_adjust is None:
priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
new_request.priority = request.priority + priority_adjust
if callable(reason):
@ -110,42 +109,51 @@ def get_retry_request(
if isinstance(reason, Exception):
reason = global_object_name(reason.__class__)
stats.inc_value(f'{stats_base_key}/count')
stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
stats.inc_value(f"{stats_base_key}/count")
stats.inc_value(f"{stats_base_key}/reason_count/{reason}")
return new_request
else:
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None
stats.inc_value(f"{stats_base_key}/max_reached")
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s",
{"request": request, "retry_times": retry_times, "reason": reason},
extra={"spider": spider},
)
return None
class RetryMiddleware:
# IOError is raised by the HttpCompression middleware when trying to
# decompress an empty response
EXCEPTIONS_TO_RETRY = (defer.TimeoutError, TimeoutError, DNSLookupError,
ConnectionRefusedError, ConnectionDone, ConnectError,
ConnectionLost, TCPTimedOutError, ResponseFailed,
IOError, TunnelError)
EXCEPTIONS_TO_RETRY = (
defer.TimeoutError,
TimeoutError,
DNSLookupError,
ConnectionRefusedError,
ConnectionDone,
ConnectError,
ConnectionLost,
TCPTimedOutError,
ResponseFailed,
IOError,
TunnelError,
)
def __init__(self, settings):
if not settings.getbool('RETRY_ENABLED'):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint('RETRY_TIMES')
self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
self.max_retry_times = settings.getint("RETRY_TIMES")
self.retry_http_codes = set(
int(x) for x in settings.getlist("RETRY_HTTP_CODES")
)
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings)
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
if request.meta.get("dont_retry", False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
@ -153,15 +161,14 @@ class RetryMiddleware:
return response
def process_exception(self, request, exception, spider):
if (
isinstance(exception, self.EXCEPTIONS_TO_RETRY)
and not request.meta.get('dont_retry', False)
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
"dont_retry", False
):
return self._retry(request, exception, spider)
def _retry(self, request, reason, spider):
max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
return get_retry_request(
request,
reason=reason,

56
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -7,8 +7,10 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
import logging
from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
@ -20,23 +22,25 @@ class RobotsTxtMiddleware:
DOWNLOAD_PRIORITY = 1000
def __init__(self, crawler):
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
raise NotConfigured
self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy")
self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None)
self.crawler = crawler
self._parsers = {}
self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER"))
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b'')
self._parserimpl.from_crawler(self.crawler, b"")
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request, spider):
if request.meta.get('dont_obey_robotstxt'):
if request.meta.get("dont_obey_robotstxt"):
return
if request.url.startswith("data:") or request.url.startswith("file:"):
return
d = maybeDeferred(self.robot_parser, request, spider)
d.addCallback(self.process_request_2, request, spider)
@ -48,11 +52,14 @@ class RobotsTxtMiddleware:
useragent = self._robotstxt_useragent
if not useragent:
useragent = request.headers.get(b'User-Agent', self._default_useragent)
useragent = request.headers.get(b"User-Agent", self._default_useragent)
if not rp.allowed(request.url, useragent):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
self.crawler.stats.inc_value('robotstxt/forbidden')
logger.debug(
"Forbidden by robots.txt: %(request)s",
{"request": request},
extra={"spider": spider},
)
self.crawler.stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
def robot_parser(self, request, spider):
@ -65,13 +72,14 @@ class RobotsTxtMiddleware:
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
meta={'dont_obey_robotstxt': True}
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value('robotstxt/request_count')
self.crawler.stats.inc_value("robotstxt/request_count")
if isinstance(self._parsers[netloc], Deferred):
d = Deferred()
@ -79,22 +87,26 @@ class RobotsTxtMiddleware:
def cb(result):
d.callback(result)
return result
self._parsers[netloc].addCallback(cb)
return d
else:
return self._parsers[netloc]
return self._parsers[netloc]
def _logerror(self, failure, request, spider):
if failure.type is not IgnoreRequest:
logger.error("Error downloading %(request)s: %(f_exception)s",
{'request': request, 'f_exception': failure.value},
exc_info=failure_to_exc_info(failure),
extra={'spider': spider})
logger.error(
"Error downloading %(request)s: %(f_exception)s",
{"request": request, "f_exception": failure.value},
exc_info=failure_to_exc_info(failure),
extra={"spider": spider},
)
return failure
def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}')
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
@ -102,7 +114,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, failure, netloc):
if failure.type is not IgnoreRequest:
key = f'robotstxt/exception_count/{failure.type}'
key = f"robotstxt/exception_count/{failure.type}"
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = None

40
scrapy/downloadermiddlewares/stats.py
View File

@ -1,9 +1,9 @@
from twisted.web import http
from scrapy.exceptions import NotConfigured
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
from twisted.web import http
def get_header_size(headers):
size = 0
@ -11,40 +11,50 @@ def get_header_size(headers):
if isinstance(value, (list, tuple)):
for v in value:
size += len(b": ") + len(key) + len(v)
return size + len(b'\r\n') * (len(headers.keys()) - 1)
return size + len(b"\r\n") * (len(headers.keys()) - 1)
def get_status_size(response_status):
return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
class DownloaderStats:
def __init__(self, stats):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('DOWNLOADER_STATS'):
if not crawler.settings.getbool("DOWNLOADER_STATS"):
raise NotConfigured
return cls(crawler.stats)
def process_request(self, request, spider):
self.stats.inc_value('downloader/request_count', spider=spider)
self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
self.stats.inc_value("downloader/request_count", spider=spider)
self.stats.inc_value(
f"downloader/request_method_count/{request.method}", spider=spider
)
reqlen = len(request_httprepr(request))
self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
def process_response(self, request, response, spider):
self.stats.inc_value('downloader/response_count', spider=spider)
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
self.stats.inc_value("downloader/response_count", spider=spider)
self.stats.inc_value(
f"downloader/response_status_count/{response.status}", spider=spider
)
reslen = (
len(response.body)
+ get_header_size(response.headers)
+ get_status_size(response.status)
+ 4
)
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
return response
def process_exception(self, request, exception, spider):
ex_class = global_object_name(exception.__class__)
self.stats.inc_value('downloader/exception_count', spider=spider)
self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)
self.stats.inc_value("downloader/exception_count", spider=spider)
self.stats.inc_value(
f"downloader/exception_type_count/{ex_class}", spider=spider
)

8
scrapy/downloadermiddlewares/useragent.py
View File

@ -6,18 +6,18 @@ from scrapy import signals
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""
def __init__(self, user_agent='Scrapy'):
def __init__(self, user_agent="Scrapy"):
self.user_agent = user_agent
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.settings['USER_AGENT'])
o = cls(crawler.settings["USER_AGENT"])
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
self.user_agent = getattr(spider, 'user_agent', self.user_agent)
self.user_agent = getattr(spider, "user_agent", self.user_agent)
def process_request(self, request, spider):
if self.user_agent:
request.headers.setdefault(b'User-Agent', self.user_agent)
request.headers.setdefault(b"User-Agent", self.user_agent)

35
scrapy/dupefilters.py
View File

@ -1,5 +1,5 @@
import logging
import os
from pathlib import Path
from typing import Optional, Set, Type, TypeVar
from warnings import warn
@ -10,15 +10,16 @@ from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, RequestFingerprinter
from scrapy.utils.request import RequestFingerprinter, referer_str
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
class BaseDupeFilter:
@classmethod
def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV:
def from_settings(
cls: Type[BaseDupeFilterTV], settings: BaseSettings
) -> BaseDupeFilterTV:
return cls()
def request_seen(self, request: Request) -> bool:
@ -55,13 +56,15 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
self.file = open(os.path.join(path, 'requests.seen'), 'a+')
self.file = Path(path, "requests.seen").open("a+", encoding="utf-8")
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
def from_settings(
cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None
) -> RFPDupeFilterTV:
debug = settings.getbool("DUPEFILTER_DEBUG")
try:
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
except TypeError:
@ -100,7 +103,7 @@ class RFPDupeFilter(BaseDupeFilter):
return True
self.fingerprints.add(fp)
if self.file:
self.file.write(fp + '\n')
self.file.write(fp + "\n")
return False
def request_fingerprint(self, request: Request) -> str:
@ -113,13 +116,15 @@ class RFPDupeFilter(BaseDupeFilter):
def log(self, request: Request, spider: Spider) -> None:
if self.debug:
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
args = {'request': request, 'referer': referer_str(request)}
self.logger.debug(msg, args, extra={'spider': spider})
args = {"request": request, "referer": referer_str(request)}
self.logger.debug(msg, args, extra={"spider": spider})
elif self.logdupes:
msg = ("Filtered duplicate request: %(request)s"
" - no more duplicates will be shown"
" (see DUPEFILTER_DEBUG to show all duplicates)")
self.logger.debug(msg, {'request': request}, extra={'spider': spider})
msg = (
"Filtered duplicate request: %(request)s"
" - no more duplicates will be shown"
" (see DUPEFILTER_DEBUG to show all duplicates)"
)
self.logger.debug(msg, {"request": request}, extra={"spider": spider})
self.logdupes = False
spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider)

11
scrapy/exceptions.py
View File

@ -10,6 +10,7 @@ new exceptions here without documenting them there.
class NotConfigured(Exception):
"""Indicates a missing configuration situation"""
pass
@ -18,6 +19,7 @@ class _InvalidOutput(TypeError):
Indicates an invalid value has been returned by a middleware's processing method.
Internal and undocumented, it should not be raised or caught by user code.
"""
pass
@ -30,13 +32,14 @@ class IgnoreRequest(Exception):
class DontCloseSpider(Exception):
"""Request the spider not to be closed yet"""
pass
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
def __init__(self, reason='cancelled'):
def __init__(self, reason="cancelled"):
super().__init__()
self.reason = reason
@ -58,11 +61,13 @@ class StopDownload(Exception):
class DropItem(Exception):
"""Drop item from the item pipeline"""
pass
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
pass
@ -73,7 +78,7 @@ class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
self.print_help = kw.pop("print_help", True)
super().__init__(*a, **kw)
@ -81,9 +86,11 @@ class ScrapyDeprecationWarning(Warning):
"""Warning category for deprecated features, since the default
DeprecationWarning is silenced on Python 2.7+
"""
pass
class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""
pass

101
scrapy/exporters.py
View File

@ -11,21 +11,26 @@ import warnings
from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',
'JsonItemExporter', 'MarshalItemExporter']
__all__ = [
"BaseItemExporter",
"PprintItemExporter",
"PickleItemExporter",
"CsvItemExporter",
"XmlItemExporter",
"JsonLinesItemExporter",
"JsonItemExporter",
"MarshalItemExporter",
]
class BaseItemExporter:
def __init__(self, *, dont_fail=False, **kwargs):
self._kwargs = kwargs
self._configure(kwargs, dont_fail=dont_fail)
@ -35,10 +40,10 @@ class BaseItemExporter:
If dont_fail is set, it won't raise an exception on unexpected options
(useful for using with keyword arguments in subclasses ``__init__`` methods)
"""
self.encoding = options.pop('encoding', None)
self.fields_to_export = options.pop('fields_to_export', None)
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent = options.pop('indent', None)
self.encoding = options.pop("encoding", None)
self.fields_to_export = options.pop("fields_to_export", None)
self.export_empty_fields = options.pop("export_empty_fields", False)
self.indent = options.pop("indent", None)
if not dont_fail and options:
raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
@ -46,7 +51,7 @@ class BaseItemExporter:
raise NotImplementedError
def serialize_field(self, field, name, value):
serializer = field.get('serializer', lambda x: x)
serializer = field.get("serializer", lambda x: x)
return serializer(value)
def start_exporting(self):
@ -74,8 +79,7 @@ class BaseItemExporter:
field_iter = self.fields_to_export.items()
else:
field_iter = (
(x, y) for x, y in self.fields_to_export.items()
if x in item
(x, y) for x, y in self.fields_to_export.items() if x in item
)
else:
if include_empty:
@ -98,36 +102,36 @@ class BaseItemExporter:
class JsonLinesItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
super().__init__(dont_fail=True, **kwargs)
self.file = file
self._kwargs.setdefault('ensure_ascii', not self.encoding)
self._kwargs.setdefault("ensure_ascii", not self.encoding)
self.encoder = ScrapyJSONEncoder(**self._kwargs)
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
data = self.encoder.encode(itemdict) + '\n'
data = self.encoder.encode(itemdict) + "\n"
self.file.write(to_bytes(data, self.encoding))
class JsonItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
super().__init__(dont_fail=True, **kwargs)
self.file = file
# there is a small difference between the behaviour or JsonItemExporter.indent
# and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
# the addition of newlines everywhere
json_indent = self.indent if self.indent is not None and self.indent > 0 else None
self._kwargs.setdefault('indent', json_indent)
self._kwargs.setdefault('ensure_ascii', not self.encoding)
json_indent = (
self.indent if self.indent is not None and self.indent > 0 else None
)
self._kwargs.setdefault("indent", json_indent)
self._kwargs.setdefault("ensure_ascii", not self.encoding)
self.encoder = ScrapyJSONEncoder(**self._kwargs)
self.first_item = True
def _beautify_newline(self):
if self.indent is not None:
self.file.write(b'\n')
self.file.write(b"\n")
def start_exporting(self):
self.file.write(b"[")
@ -141,7 +145,7 @@ class JsonItemExporter(BaseItemExporter):
if self.first_item:
self.first_item = False
else:
self.file.write(b',')
self.file.write(b",")
self._beautify_newline()
itemdict = dict(self._get_serialized_fields(item))
data = self.encoder.encode(itemdict)
@ -149,22 +153,21 @@ class JsonItemExporter(BaseItemExporter):
class XmlItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
self.item_element = kwargs.pop('item_element', 'item')
self.root_element = kwargs.pop('root_element', 'items')
self.item_element = kwargs.pop("item_element", "item")
self.root_element = kwargs.pop("root_element", "items")
super().__init__(**kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.encoding = "utf-8"
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
self.xg.characters('\n')
self.xg.characters("\n")
def _beautify_indent(self, depth=1):
if self.indent:
self.xg.characters(' ' * self.indent * depth)
self.xg.characters(" " * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
@ -175,7 +178,7 @@ class XmlItemExporter(BaseItemExporter):
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, {})
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=''):
for name, value in self._get_serialized_fields(item, default_value=""):
self._export_xml_field(name, value, depth=2)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
@ -188,7 +191,7 @@ class XmlItemExporter(BaseItemExporter):
def _export_xml_field(self, name, serialized_value, depth):
self._beautify_indent(depth=depth)
self.xg.startElement(name, {})
if hasattr(serialized_value, 'items'):
if hasattr(serialized_value, "items"):
self._beautify_newline()
for subname, value in serialized_value.items():
self._export_xml_field(subname, value, depth=depth + 1)
@ -196,7 +199,7 @@ class XmlItemExporter(BaseItemExporter):
elif is_listlike(serialized_value):
self._beautify_newline()
for value in serialized_value:
self._export_xml_field('value', value, depth=depth + 1)
self._export_xml_field("value", value, depth=depth + 1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, str):
self.xg.characters(serialized_value)
@ -207,18 +210,24 @@ class XmlItemExporter(BaseItemExporter):
class CsvItemExporter(BaseItemExporter):
def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
def __init__(
self,
file,
include_headers_line=True,
join_multivalued=",",
errors=None,
**kwargs,
):
super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.encoding = "utf-8"
self.include_headers_line = include_headers_line
self.stream = io.TextIOWrapper(
file,
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034
newline="", # Windows needs this https://github.com/scrapy/scrapy/issues/3034
errors=errors,
)
self.csv_writer = csv.writer(self.stream, **self._kwargs)
@ -226,7 +235,7 @@ class CsvItemExporter(BaseItemExporter):
self._join_multivalued = join_multivalued
def serialize_field(self, field, name, value):
serializer = field.get('serializer', self._join_if_needed)
serializer = field.get("serializer", self._join_if_needed)
return serializer(value)
def _join_if_needed(self, value):
@ -242,8 +251,7 @@ class CsvItemExporter(BaseItemExporter):
self._headers_not_written = False
self._write_headers_and_set_fields_to_export(item)
fields = self._get_serialized_fields(item, default_value='',
include_empty=True)
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
values = list(self._build_row(x for _, x in fields))
self.csv_writer.writerow(values)
@ -271,7 +279,6 @@ class CsvItemExporter(BaseItemExporter):
class PickleItemExporter(BaseItemExporter):
def __init__(self, file, protocol=4, **kwargs):
super().__init__(**kwargs)
self.file = file
@ -300,14 +307,13 @@ class MarshalItemExporter(BaseItemExporter):
class PprintItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
super().__init__(**kwargs)
self.file = file
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))
self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
class PythonItemExporter(BaseItemExporter):
@ -321,25 +327,26 @@ class PythonItemExporter(BaseItemExporter):
"""
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
self.binary = options.pop("binary", True)
super()._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",
ScrapyDeprecationWarning)
ScrapyDeprecationWarning,
)
if not self.encoding:
self.encoding = 'utf-8'
self.encoding = "utf-8"
def serialize_field(self, field, name, value):
serializer = field.get('serializer', self._serialize_value)
serializer = field.get("serializer", self._serialize_value)
return serializer(value)
def _serialize_value(self, value):
if isinstance(value, Item):
return self.export_item(value)
elif is_item(value):
if is_item(value):
return dict(self._serialize_item(value))
elif is_listlike(value):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (str, bytes)):

4
scrapy/extension.py
View File

@ -9,8 +9,8 @@ from scrapy.utils.conf import build_component_list
class ExtensionManager(MiddlewareManager):
component_name = 'extension'
component_name = "extension"
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(settings.getwithbase('EXTENSIONS'))
return build_component_list(settings.getwithbase("EXTENSIONS"))

47
scrapy/extensions/closespider.py
View File

@ -11,15 +11,14 @@ from scrapy.exceptions import NotConfigured
class CloseSpider:
def __init__(self, crawler):
self.crawler = crawler
self.close_on = {
'timeout': crawler.settings.getfloat('CLOSESPIDER_TIMEOUT'),
'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
"timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"),
"itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
}
if not any(self.close_on.values()):
@ -27,13 +26,13 @@ class CloseSpider:
self.counter = defaultdict(int)
if self.close_on.get('errorcount'):
if self.close_on.get("errorcount"):
crawler.signals.connect(self.error_count, signal=signals.spider_error)
if self.close_on.get('pagecount'):
if self.close_on.get("pagecount"):
crawler.signals.connect(self.page_count, signal=signals.response_received)
if self.close_on.get('timeout'):
if self.close_on.get("timeout"):
crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
if self.close_on.get('itemcount'):
if self.close_on.get("itemcount"):
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@ -42,27 +41,31 @@ class CloseSpider:
return cls(crawler)
def error_count(self, failure, response, spider):
self.counter['errorcount'] += 1
if self.counter['errorcount'] == self.close_on['errorcount']:
self.crawler.engine.close_spider(spider, 'closespider_errorcount')
self.counter["errorcount"] += 1
if self.counter["errorcount"] == self.close_on["errorcount"]:
self.crawler.engine.close_spider(spider, "closespider_errorcount")
def page_count(self, response, request, spider):
self.counter['pagecount'] += 1
if self.counter['pagecount'] == self.close_on['pagecount']:
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
self.counter["pagecount"] += 1
if self.counter["pagecount"] == self.close_on["pagecount"]:
self.crawler.engine.close_spider(spider, "closespider_pagecount")
def spider_opened(self, spider):
from twisted.internet import reactor
self.task = reactor.callLater(self.close_on['timeout'],
self.crawler.engine.close_spider, spider,
reason='closespider_timeout')
self.task = reactor.callLater(
self.close_on["timeout"],
self.crawler.engine.close_spider,
spider,
reason="closespider_timeout",
)
def item_scraped(self, item, spider):
self.counter['itemcount'] += 1
if self.counter['itemcount'] == self.close_on['itemcount']:
self.crawler.engine.close_spider(spider, 'closespider_itemcount')
self.counter["itemcount"] += 1
if self.counter["itemcount"] == self.close_on["itemcount"]:
self.crawler.engine.close_spider(spider, "closespider_itemcount")
def spider_closed(self, spider):
task = getattr(self, 'task', False)
task = getattr(self, "task", False)
if task and task.active():
task.cancel()

19
scrapy/extensions/corestats.py
View File

@ -7,7 +7,6 @@ from scrapy import signals
class CoreStats:
def __init__(self, stats):
self.stats = stats
self.start_time = None
@ -24,23 +23,25 @@ class CoreStats:
def spider_opened(self, spider):
self.start_time = datetime.utcnow()
self.stats.set_value('start_time', self.start_time, spider=spider)
self.stats.set_value("start_time", self.start_time, spider=spider)
def spider_closed(self, spider, reason):
finish_time = datetime.utcnow()
elapsed_time = finish_time - self.start_time
elapsed_time_seconds = elapsed_time.total_seconds()
self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
self.stats.set_value('finish_time', finish_time, spider=spider)
self.stats.set_value('finish_reason', reason, spider=spider)
self.stats.set_value(
"elapsed_time_seconds", elapsed_time_seconds, spider=spider
)
self.stats.set_value("finish_time", finish_time, spider=spider)
self.stats.set_value("finish_reason", reason, spider=spider)
def item_scraped(self, item, spider):
self.stats.inc_value('item_scraped_count', spider=spider)
self.stats.inc_value("item_scraped_count", spider=spider)
def response_received(self, spider):
self.stats.inc_value('response_received_count', spider=spider)
self.stats.inc_value("response_received_count", spider=spider)
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
self.stats.inc_value('item_dropped_count', spider=spider)
self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)
self.stats.inc_value("item_dropped_count", spider=spider)
self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)

28
scrapy/extensions/debug.py
View File

@ -4,11 +4,11 @@ Extensions for debugging Scrapy
See documentation in docs/topics/extensions.rst
"""
import sys
import signal
import logging
import traceback
import signal
import sys
import threading
import traceback
from pdb import Pdb
from scrapy.utils.engine import format_engine_status
@ -18,7 +18,6 @@ logger = logging.getLogger(__name__)
class StackTraceDump:
def __init__(self, crawler=None):
self.crawler = crawler
try:
@ -34,20 +33,23 @@ class StackTraceDump:
def dump_stacktrace(self, signum, frame):
log_args = {
'stackdumps': self._thread_stacks(),
'enginestatus': format_engine_status(self.crawler.engine),
'liverefs': format_live_refs(),
"stackdumps": self._thread_stacks(),
"enginestatus": format_engine_status(self.crawler.engine),
"liverefs": format_live_refs(),
}
logger.info("Dumping stack trace and engine status\n"
"%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s",
log_args, extra={'crawler': self.crawler})
logger.info(
"Dumping stack trace and engine status\n"
"%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s",
log_args,
extra={"crawler": self.crawler},
)
def _thread_stacks(self):
id2name = dict((th.ident, th.name) for th in threading.enumerate())
dumps = ''
dumps = ""
for id_, frame in sys._current_frames().items():
name = id2name.get(id_, '')
dump = ''.join(traceback.format_stack(frame))
name = id2name.get(id_, "")
dump = "".join(traceback.format_stack(frame))
dumps += f"# Thread: {name}({id_})\n{dump}\n"
return dumps

318
scrapy/extensions/feedexport.py
View File

@ -5,20 +5,20 @@ See documentation in docs/topics/feed-exports.rst
"""
import logging
import os
import re
import sys
import warnings
from datetime import datetime
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, Callable, Optional, Tuple, Union
from typing import IO, Any, Callable, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
from w3lib.url import file_uri_to_path
from zope.interface import implementer, Interface
from zope.interface import Interface, implementer
from scrapy import signals, Spider
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.utils.boto import is_botocore_available
@ -28,21 +28,20 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
if 'feed_options' in argument_names:
kwargs['feed_options'] = feed_options
if "feed_options" in argument_names:
kwargs["feed_options"] = feed_options
else:
warnings.warn(
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy.",
category=ScrapyDeprecationWarning
category=ScrapyDeprecationWarning,
)
return builder(*preargs, uri, *args, **kwargs)
@ -55,6 +54,7 @@ class ItemFilter:
:param feed_options: feed specific options passed from FeedExporter
:type feed_options: dict
"""
feed_options: Optional[dict]
item_classes: Tuple
@ -62,7 +62,8 @@ class ItemFilter:
self.feed_options = feed_options
if feed_options is not None:
self.item_classes = tuple(
load_object(item_class) for item_class in feed_options.get("item_classes") or ()
load_object(item_class)
for item_class in feed_options.get("item_classes") or ()
)
else:
self.item_classes = tuple()
@ -98,13 +99,12 @@ class IFeedStorage(Interface):
@implementer(IFeedStorage)
class BlockingFeedStorage:
def open(self, spider):
path = spider.crawler.settings['FEED_TEMPDIR']
if path and not os.path.isdir(path):
raise OSError('Not a Directory: ' + str(path))
path = spider.crawler.settings["FEED_TEMPDIR"]
if path and not Path(path).is_dir():
raise OSError("Not a Directory: " + str(path))
return NamedTemporaryFile(prefix='feed-', dir=path)
return NamedTemporaryFile(prefix="feed-", dir=path)
def store(self, file):
return threads.deferToThread(self._store_in_thread, file)
@ -115,16 +115,17 @@ class BlockingFeedStorage:
@implementer(IFeedStorage)
class StdoutFeedStorage:
def __init__(self, uri, _stdout=None, *, feed_options=None):
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout = _stdout
if feed_options and feed_options.get('overwrite', False) is True:
logger.warning('Standard output (stdout) storage does not support '
'overwriting. To suppress this warning, remove the '
'overwrite option from your FEEDS setting, or set '
'it to False.')
if feed_options and feed_options.get("overwrite", False) is True:
logger.warning(
"Standard output (stdout) storage does not support "
"overwriting. To suppress this warning, remove the "
"overwrite option from your FEEDS setting, or set "
"it to False."
)
def open(self, spider):
return self._stdout
@ -135,28 +136,35 @@ class StdoutFeedStorage:
@implementer(IFeedStorage)
class FileFeedStorage:
def __init__(self, uri, *, feed_options=None):
self.path = file_uri_to_path(uri)
feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab"
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
return open(self.path, self.write_mode)
def open(self, spider) -> IO[Any]:
dirname = Path(self.path).parent
if dirname and not dirname.exists():
dirname.mkdir(parents=True)
return Path(self.path).open(self.write_mode)
def store(self, file):
file.close()
class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
feed_options=None, session_token=None):
def __init__(
self,
uri,
access_key=None,
secret_key=None,
acl=None,
endpoint_url=None,
*,
feed_options=None,
session_token=None,
):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
raise NotConfigured("missing botocore library")
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
@ -166,41 +174,45 @@ class S3FeedStorage(BlockingFeedStorage):
self.acl = acl
self.endpoint_url = endpoint_url
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
'suppress this warning, remove the overwrite '
'option from your FEEDS setting or set it to True.')
endpoint_url=self.endpoint_url,
)
if feed_options and feed_options.get("overwrite", True) is False:
logger.warning(
"S3 does not support appending to files. To "
"suppress this warning, remove the overwrite "
"option from your FEEDS setting or set it to True."
)
@classmethod
def from_crawler(cls, crawler, uri, *, feed_options=None):
return build_storage(
cls,
uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
session_token=crawler.settings['AWS_SESSION_TOKEN'],
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
access_key=crawler.settings["AWS_ACCESS_KEY_ID"],
secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"],
session_token=crawler.settings["AWS_SESSION_TOKEN"],
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
feed_options=feed_options,
)
def _store_in_thread(self, file):
file.seek(0)
kwargs = {'ACL': self.acl} if self.acl else {}
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file,
**kwargs)
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
file.close()
class GCSFeedStorage(BlockingFeedStorage):
def __init__(self, uri, project_id, acl):
self.project_id = project_id
self.acl = acl
@ -212,13 +224,14 @@ class GCSFeedStorage(BlockingFeedStorage):
def from_crawler(cls, crawler, uri):
return cls(
uri,
crawler.settings['GCS_PROJECT_ID'],
crawler.settings['FEED_STORAGE_GCS_ACL'] or None
crawler.settings["GCS_PROJECT_ID"],
crawler.settings["FEED_STORAGE_GCS_ACL"] or None,
)
def _store_in_thread(self, file):
file.seek(0)
from google.cloud.storage import Client
client = Client(project=self.project_id)
bucket = client.get_bucket(self.bucket_name)
blob = bucket.blob(self.blob_name)
@ -226,37 +239,51 @@ class GCSFeedStorage(BlockingFeedStorage):
class FTPFeedStorage(BlockingFeedStorage):
def __init__(self, uri, use_active_mode=False, *, feed_options=None):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
self.port = int(u.port or "21")
self.username = u.username
self.password = unquote(u.password or '')
self.password = unquote(u.password or "")
self.path = u.path
self.use_active_mode = use_active_mode
self.overwrite = not feed_options or feed_options.get('overwrite', True)
self.overwrite = not feed_options or feed_options.get("overwrite", True)
@classmethod
def from_crawler(cls, crawler, uri, *, feed_options=None):
return build_storage(
cls,
uri,
crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"),
feed_options=feed_options,
)
def _store_in_thread(self, file):
ftp_store_file(
path=self.path, file=file, host=self.host,
port=self.port, username=self.username,
password=self.password, use_active_mode=self.use_active_mode,
path=self.path,
file=file,
host=self.host,
port=self.port,
username=self.username,
password=self.password,
use_active_mode=self.use_active_mode,
overwrite=self.overwrite,
)
class _FeedSlot:
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter):
def __init__(
self,
file,
exporter,
storage,
uri,
format,
store_empty,
batch_id,
uri_template,
filter,
):
self.file = file
self.exporter = exporter
self.storage = storage
@ -283,7 +310,6 @@ class _FeedSlot:
class FeedExporter:
@classmethod
def from_crawler(cls, crawler):
exporter = cls(crawler)
@ -299,48 +325,55 @@ class FeedExporter:
self.slots = []
self.filters = {}
if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
raise NotConfigured
# Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings
if self.settings['FEED_URI']:
if self.settings["FEED_URI"]:
warnings.warn(
'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of '
'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details',
category=ScrapyDeprecationWarning, stacklevel=2,
"The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of "
"the `FEEDS` setting. Please see the `FEEDS` setting docs for more details",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects
feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
)
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.filters[uri] = self._load_filter(feed_options)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed_options in self.settings.getdict('FEEDS').items():
for uri, feed_options in self.settings.getdict("FEEDS").items():
uri = str(uri) # handle pathlib.Path objects
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
)
self.filters[uri] = self._load_filter(feed_options)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
self.storages = self._load_components("FEED_STORAGES")
self.exporters = self._load_components("FEED_EXPORTERS")
for uri, feed_options in self.feeds.items():
if not self._storage_supported(uri, feed_options):
raise NotConfigured
if not self._settings_are_valid():
raise NotConfigured
if not self._exporter_supported(feed_options['format']):
if not self._exporter_supported(feed_options["format"]):
raise NotConfigured
def open_spider(self, spider):
for uri, feed_options in self.feeds.items():
uri_params = self._get_uri_params(spider, feed_options['uri_params'])
self.slots.append(self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
feed_options=feed_options,
spider=spider,
uri_template=uri,
))
uri_params = self._get_uri_params(spider, feed_options["uri_params"])
self.slots.append(
self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
feed_options=feed_options,
spider=spider,
uri_template=uri,
)
)
def close_spider(self, spider):
deferred_list = []
@ -350,11 +383,11 @@ class FeedExporter:
return defer.DeferredList(deferred_list) if deferred_list else None
def _close_slot(self, slot, spider):
slot.finish_exporting()
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
return defer.maybeDeferred(slot.storage.store, slot.file)
slot.finish_exporting()
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d = defer.maybeDeferred(slot.storage.store, slot.file)
@ -368,16 +401,15 @@ class FeedExporter:
def _handle_store_error(self, f, logmsg, spider, slot_type):
logger.error(
"Error storing %s", logmsg,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
"Error storing %s",
logmsg,
exc_info=failure_to_exc_info(f),
extra={"spider": spider},
)
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
def _handle_store_success(self, f, logmsg, spider, slot_type):
logger.info(
"Stored %s", logmsg,
extra={'spider': spider}
)
logger.info("Stored %s", logmsg, extra={"spider": spider})
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
@ -393,26 +425,28 @@ class FeedExporter:
storage = self._get_storage(uri, feed_options)
file = storage.open(spider)
if "postprocessing" in feed_options:
file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)
file = PostProcessingManager(
feed_options["postprocessing"], file, feed_options
)
exporter = self._get_exporter(
file=file,
format=feed_options['format'],
fields_to_export=feed_options['fields'],
encoding=feed_options['encoding'],
indent=feed_options['indent'],
**feed_options['item_export_kwargs'],
format=feed_options["format"],
fields_to_export=feed_options["fields"],
encoding=feed_options["encoding"],
indent=feed_options["indent"],
**feed_options["item_export_kwargs"],
)
slot = _FeedSlot(
file=file,
exporter=exporter,
storage=storage,
uri=uri,
format=feed_options['format'],
store_empty=feed_options['store_empty'],
format=feed_options["format"],
store_empty=feed_options["store_empty"],
batch_id=batch_id,
uri_template=uri_template,
filter=self.filters[uri_template]
filter=self.filters[uri_template],
)
if slot.store_empty:
slot.start_exporting()
@ -422,7 +456,9 @@ class FeedExporter:
slots = []
for slot in self.slots:
if not slot.filter.accepts(item):
slots.append(slot) # if slot doesn't accept item, continue with next slot
slots.append(
slot
) # if slot doesn't accept item, continue with next slot
continue
slot.start_exporting()
@ -430,18 +466,22 @@ class FeedExporter:
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
if (
self.feeds[slot.uri_template]['batch_item_count']
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
self.feeds[slot.uri_template]["batch_item_count"]
and slot.itemcount >= self.feeds[slot.uri_template]["batch_item_count"]
):
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
uri_params = self._get_uri_params(
spider, self.feeds[slot.uri_template]["uri_params"], slot
)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
feed_options=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,
))
slots.append(
self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
feed_options=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,
)
)
else:
slots.append(slot)
self.slots = slots
@ -459,7 +499,7 @@ class FeedExporter:
def _exporter_supported(self, format):
if format in self.exporters:
return True
logger.error("Unknown feed format: %(format)s", {'format': format})
logger.error("Unknown feed format: %(format)s", {"format": format})
def _settings_are_valid(self):
"""
@ -467,12 +507,14 @@ class FeedExporter:
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
"""
for uri_template, values in self.feeds.items():
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
if values["batch_item_count"] and not re.search(
r"%\(batch_time\)s|%\(batch_id\)", uri_template
):
logger.error(
'%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
uri_template
"%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT "
"setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: "
"https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count",
uri_template,
)
return False
return True
@ -484,17 +526,17 @@ class FeedExporter:
self._get_storage(uri, feed_options)
return True
except NotConfigured as e:
logger.error("Disabled feed storage scheme: %(scheme)s. "
"Reason: %(reason)s",
{'scheme': scheme, 'reason': str(e)})
logger.error(
"Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s",
{"scheme": scheme, "reason": str(e)},
)
else:
logger.error("Unknown feed storage scheme: %(scheme)s",
{'scheme': scheme})
logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})
def _get_instance(self, objcls, *args, **kwargs):
return create_instance(
objcls, self.settings, getattr(self, 'crawler', None),
*args, **kwargs)
objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs
)
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
@ -506,20 +548,22 @@ class FeedExporter:
do not support it, and issuing a deprecation warning instead.
"""
feedcls = self.storages[urlparse(uri).scheme]
crawler = getattr(self, 'crawler', None)
crawler = getattr(self, "crawler", None)
def build_instance(builder, *preargs):
return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
return build_storage(
builder, uri, feed_options=feed_options, preargs=preargs
)
if crawler and hasattr(feedcls, 'from_crawler'):
if crawler and hasattr(feedcls, "from_crawler"):
instance = build_instance(feedcls.from_crawler, crawler)
method_name = 'from_crawler'
elif hasattr(feedcls, 'from_settings'):
method_name = "from_crawler"
elif hasattr(feedcls, "from_settings"):
instance = build_instance(feedcls.from_settings, self.settings)
method_name = 'from_settings'
method_name = "from_settings"
else:
instance = build_instance(feedcls)
method_name = '__new__'
method_name = "__new__"
if instance is None:
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
return instance
@ -534,19 +578,23 @@ class FeedExporter:
for k in dir(spider):
params[k] = getattr(spider, k)
utc_now = datetime.utcnow()
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
params['batch_time'] = utc_now.isoformat().replace(':', '-')
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
params["batch_time"] = utc_now.isoformat().replace(":", "-")
params["batch_id"] = slot.batch_id + 1 if slot is not None else 1
original_params = params.copy()
uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
uripar_function = (
load_object(uri_params_function)
if uri_params_function
else lambda params, _: params
)
new_params = uripar_function(params, spider)
if new_params is None or original_params != params:
warnings.warn(
'Modifying the params dictionary in-place in the function defined in '
'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
'setting is deprecated. The function must return a new dictionary '
'instead.',
category=ScrapyDeprecationWarning
"Modifying the params dictionary in-place in the function defined in "
"the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS "
"setting is deprecated. The function must return a new dictionary "
"instead.",
category=ScrapyDeprecationWarning,
)
return new_params if new_params is not None else params

205
scrapy/extensions/httpcache.py
View File

@ -1,29 +1,31 @@
import gzip
import logging
import os
import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
from time import time
from weakref import WeakKeyDictionary
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
from scrapy.http import Headers, Response
from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
logger = logging.getLogger(__name__)
class DummyPolicy:
def __init__(self, settings):
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.ignore_http_codes = [int(x) for x in settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')]
self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
self.ignore_http_codes = [
int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES")
]
def should_cache_request(self, request):
return urlparse_cached(request).scheme not in self.ignore_schemes
@ -43,16 +45,17 @@ class RFC2616Policy:
MAXAGE = 3600 * 24 * 365 # one year
def __init__(self, settings):
self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE")
self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
self._cc_parsed = WeakKeyDictionary()
self.ignore_response_cache_controls = [
to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
to_bytes(cc)
for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS")
]
def _parse_cachecontrol(self, r):
if r not in self._cc_parsed:
cch = r.headers.get(b'Cache-Control', b'')
cch = r.headers.get(b"Cache-Control", b"")
parsed = parse_cachecontrol(cch)
if isinstance(r, Response):
for key in self.ignore_response_cache_controls:
@ -65,7 +68,7 @@ class RFC2616Policy:
return False
cc = self._parse_cachecontrol(request)
# obey user-agent directive "Cache-Control: no-store"
if b'no-store' in cc:
if b"no-store" in cc:
return False
# Any other is eligible for caching
return True
@ -76,37 +79,38 @@ class RFC2616Policy:
# Status code 206 is not included because cache can not deal with partial contents
cc = self._parse_cachecontrol(response)
# obey directive "Cache-Control: no-store"
if b'no-store' in cc:
if b"no-store" in cc:
return False
# Never cache 304 (Not Modified) responses
elif response.status == 304:
if response.status == 304:
return False
# Cache unconditionally if configured to do so
elif self.always_store:
if self.always_store:
return True
# Any hint on response expiration is good
elif b'max-age' in cc or b'Expires' in response.headers:
if b"max-age" in cc or b"Expires" in response.headers:
return True
# Firefox fallbacks this statuses to one year expiration if none is set
elif response.status in (300, 301, 308):
if response.status in (300, 301, 308):
return True
# Other statuses without expiration requires at least one validator
elif response.status in (200, 203, 401):
return b'Last-Modified' in response.headers or b'ETag' in response.headers
if response.status in (200, 203, 401):
return b"Last-Modified" in response.headers or b"ETag" in response.headers
# Any other is probably not eligible for caching
# Makes no sense to cache responses that does not contain expiration
# info and can not be revalidated
else:
return False
return False
def is_cached_response_fresh(self, cachedresponse, request):
cc = self._parse_cachecontrol(cachedresponse)
ccreq = self._parse_cachecontrol(request)
if b'no-cache' in cc or b'no-cache' in ccreq:
if b"no-cache" in cc or b"no-cache" in ccreq:
return False
now = time()
freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now)
freshnesslifetime = self._compute_freshness_lifetime(
cachedresponse, request, now
)
currentage = self._compute_current_age(cachedresponse, request, now)
reqmaxage = self._get_max_age(ccreq)
@ -116,7 +120,7 @@ class RFC2616Policy:
if currentage < freshnesslifetime:
return True
if b'max-stale' in ccreq and b'must-revalidate' not in cc:
if b"max-stale" in ccreq and b"must-revalidate" not in cc:
# From RFC2616: "Indicates that the client is willing to
# accept a response that has exceeded its expiration time.
# If max-stale is assigned a value, then the client is
@ -124,7 +128,7 @@ class RFC2616Policy:
# expiration time by no more than the specified number of
# seconds. If no value is assigned to max-stale, then the
# client is willing to accept a stale response of any age."
staleage = ccreq[b'max-stale']
staleage = ccreq[b"max-stale"]
if staleage is None:
return True
@ -143,22 +147,24 @@ class RFC2616Policy:
# as long as the old response didn't specify must-revalidate.
if response.status >= 500:
cc = self._parse_cachecontrol(cachedresponse)
if b'must-revalidate' not in cc:
if b"must-revalidate" not in cc:
return True
# Use the cached response if the server says it hasn't changed.
return response.status == 304
def _set_conditional_validators(self, request, cachedresponse):
if b'Last-Modified' in cachedresponse.headers:
request.headers[b'If-Modified-Since'] = cachedresponse.headers[b'Last-Modified']
if b"Last-Modified" in cachedresponse.headers:
request.headers[b"If-Modified-Since"] = cachedresponse.headers[
b"Last-Modified"
]
if b'ETag' in cachedresponse.headers:
request.headers[b'If-None-Match'] = cachedresponse.headers[b'ETag']
if b"ETag" in cachedresponse.headers:
request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"]
def _get_max_age(self, cc):
try:
return max(0, int(cc[b'max-age']))
return max(0, int(cc[b"max-age"]))
except (KeyError, ValueError):
return None
@ -171,18 +177,18 @@ class RFC2616Policy:
return maxage
# Parse date header or synthesize it if none exists
date = rfc1123_to_epoch(response.headers.get(b'Date')) or now
date = rfc1123_to_epoch(response.headers.get(b"Date")) or now
# Try HTTP/1.0 Expires header
if b'Expires' in response.headers:
expires = rfc1123_to_epoch(response.headers[b'Expires'])
if b"Expires" in response.headers:
expires = rfc1123_to_epoch(response.headers[b"Expires"])
# When parsing Expires header fails RFC 2616 section 14.21 says we
# should treat this as an expiration time in the past.
return max(0, expires - date) if expires else 0
# Fallback to heuristic using last-modified header
# This is not in RFC but on Firefox caching implementation
lastmodified = rfc1123_to_epoch(response.headers.get(b'Last-Modified'))
lastmodified = rfc1123_to_epoch(response.headers.get(b"Last-Modified"))
if lastmodified and lastmodified <= date:
return (date - lastmodified) / 10
@ -190,7 +196,7 @@ class RFC2616Policy:
if response.status in (300, 301, 308):
return self.MAXAGE
# Insufficient information to compute fresshness lifetime
# Insufficient information to compute freshness lifetime
return 0
def _compute_current_age(self, response, request, now):
@ -199,13 +205,13 @@ class RFC2616Policy:
currentage = 0
# If Date header is not set we assume it is a fast connection, and
# clock is in sync with the server
date = rfc1123_to_epoch(response.headers.get(b'Date')) or now
date = rfc1123_to_epoch(response.headers.get(b"Date")) or now
if now > date:
currentage = now - date
if b'Age' in response.headers:
if b"Age" in response.headers:
try:
age = int(response.headers[b'Age'])
age = int(response.headers[b"Age"])
currentage = max(currentage, age)
except ValueError:
pass
@ -214,18 +220,21 @@ class RFC2616Policy:
class DbmCacheStorage:
def __init__(self, settings):
self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True)
self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"])
self.db = None
def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
def open_spider(self, spider: Spider):
dbpath = Path(self.cachedir, f"{spider.name}.db")
self.db = self.dbmodule.open(str(dbpath), "c")
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
logger.debug(
"Using DBM cache storage in %(cachepath)s",
{"cachepath": dbpath},
extra={"spider": spider},
)
self._fingerprinter = spider.crawler.request_fingerprinter
@ -236,10 +245,10 @@ class DbmCacheStorage:
data = self._read_data(spider, request)
if data is None:
return # not cached
url = data['url']
status = data['status']
headers = Headers(data['headers'])
body = data['body']
url = data["url"]
status = data["status"]
headers = Headers(data["headers"])
body = data["body"]
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
@ -247,18 +256,18 @@ class DbmCacheStorage:
def store_response(self, spider, request, response):
key = self._fingerprinter.fingerprint(request).hex()
data = {
'status': response.status,
'url': response.url,
'headers': dict(response.headers),
'body': response.body,
"status": response.status,
"url": response.url,
"headers": dict(response.headers),
"body": response.body,
}
self.db[f'{key}_data'] = pickle.dumps(data, protocol=4)
self.db[f'{key}_time'] = str(time())
self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
self.db[f"{key}_time"] = str(time())
def _read_data(self, spider, request):
key = self._fingerprinter.fingerprint(request).hex()
db = self.db
tkey = f'{key}_time'
tkey = f"{key}_time"
if tkey not in db:
return # not found
@ -266,81 +275,83 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return # expired
return pickle.loads(db[f'{key}_data'])
return pickle.loads(db[f"{key}_data"])
class FilesystemCacheStorage:
def __init__(self, settings):
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
self.cachedir = data_path(settings["HTTPCACHE_DIR"])
self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.use_gzip = settings.getbool("HTTPCACHE_GZIP")
self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider):
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
def open_spider(self, spider: Spider):
logger.debug(
"Using filesystem cache storage in %(cachedir)s",
{"cachedir": self.cachedir},
extra={"spider": spider},
)
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider):
pass
def retrieve_response(self, spider, request):
def retrieve_response(self, spider: Spider, request: Request):
"""Return response if present in cache, or None otherwise."""
metadata = self._read_meta(spider, request)
if metadata is None:
return # not cached
rpath = self._get_request_path(spider, request)
with self._open(os.path.join(rpath, 'response_body'), 'rb') as f:
rpath = Path(self._get_request_path(spider, request))
with self._open(rpath / "response_body", "rb") as f:
body = f.read()
with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f:
with self._open(rpath / "response_headers", "rb") as f:
rawheaders = f.read()
url = metadata.get('response_url')
status = metadata['status']
url = metadata.get("response_url")
status = metadata["status"]
headers = Headers(headers_raw_to_dict(rawheaders))
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
def store_response(self, spider: Spider, request: Request, response):
"""Store the given response in the cache."""
rpath = self._get_request_path(spider, request)
if not os.path.exists(rpath):
os.makedirs(rpath)
rpath = Path(self._get_request_path(spider, request))
if not rpath.exists():
rpath.mkdir(parents=True)
metadata = {
'url': request.url,
'method': request.method,
'status': response.status,
'response_url': response.url,
'timestamp': time(),
"url": request.url,
"method": request.method,
"status": response.status,
"response_url": response.url,
"timestamp": time(),
}
with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
with self._open(rpath / "meta", "wb") as f:
f.write(to_bytes(repr(metadata)))
with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
with self._open(rpath / "pickled_meta", "wb") as f:
pickle.dump(metadata, f, protocol=4)
with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
with self._open(rpath / "response_headers", "wb") as f:
f.write(headers_dict_to_raw(response.headers))
with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
with self._open(rpath / "response_body", "wb") as f:
f.write(response.body)
with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f:
with self._open(rpath / "request_headers", "wb") as f:
f.write(headers_dict_to_raw(request.headers))
with self._open(os.path.join(rpath, 'request_body'), 'wb') as f:
with self._open(rpath / "request_body", "wb") as f:
f.write(request.body)
def _get_request_path(self, spider, request):
def _get_request_path(self, spider: Spider, request: Request) -> str:
key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
return str(Path(self.cachedir, spider.name, key[0:2], key))
def _read_meta(self, spider, request):
rpath = self._get_request_path(spider, request)
metapath = os.path.join(rpath, 'pickled_meta')
if not os.path.exists(metapath):
def _read_meta(self, spider: Spider, request: Request):
rpath = Path(self._get_request_path(spider, request))
metapath = rpath / "pickled_meta"
if not metapath.exists():
return # not found
mtime = os.stat(metapath).st_mtime
mtime = metapath.stat().st_mtime
if 0 < self.expiration_secs < time() - mtime:
return # expired
with self._open(metapath, 'rb') as f:
with self._open(metapath, "rb") as f:
return pickle.load(f)
@ -357,8 +368,8 @@ def parse_cachecontrol(header):
"""
directives = {}
for directive in header.split(b','):
key, sep, val = directive.strip().partition(b'=')
for directive in header.split(b","):
key, sep, val = directive.strip().partition(b"=")
if key:
directives[key.lower()] = val if sep else None
return directives
@ -366,7 +377,7 @@ def parse_cachecontrol(header):
def rfc1123_to_epoch(date_str):
try:
date_str = to_unicode(date_str, encoding='ascii')
date_str = to_unicode(date_str, encoding="ascii")
return mktime_tz(parsedate_tz(date_str))
except Exception:
return None

24
scrapy/extensions/logstats.py
View File

@ -2,8 +2,8 @@ import logging
from twisted.internet import task
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.exceptions import NotConfigured
logger = logging.getLogger(__name__)
@ -19,7 +19,7 @@ class LogStats:
@classmethod
def from_crawler(cls, crawler):
interval = crawler.settings.getfloat('LOGSTATS_INTERVAL')
interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
if not interval:
raise NotConfigured
o = cls(crawler.stats, interval)
@ -35,17 +35,23 @@ class LogStats:
self.task.start(self.interval)
def log(self, spider):
items = self.stats.get_value('item_scraped_count', 0)
pages = self.stats.get_value('response_received_count', 0)
items = self.stats.get_value("item_scraped_count", 0)
pages = self.stats.get_value("response_received_count", 0)
irate = (items - self.itemsprev) * self.multiplier
prate = (pages - self.pagesprev) * self.multiplier
self.pagesprev, self.itemsprev = pages, items
msg = ("Crawled %(pages)d pages (at %(pagerate)d pages/min), "
"scraped %(items)d items (at %(itemrate)d items/min)")
log_args = {'pages': pages, 'pagerate': prate,
'items': items, 'itemrate': irate}
logger.info(msg, log_args, extra={'spider': spider})
msg = (
"Crawled %(pages)d pages (at %(pagerate)d pages/min), "
"scraped %(items)d items (at %(itemrate)d items/min)"
)
log_args = {
"pages": pages,
"pagerate": prate,
"items": items,
"itemrate": irate,
}
logger.info(msg, log_args, extra={"spider": spider})
def spider_closed(self, spider, reason):
if self.task and self.task.running:

11
scrapy/extensions/memdebug.py
View File

@ -12,13 +12,12 @@ from scrapy.utils.trackref import live_refs
class MemoryDebugger:
def __init__(self, stats):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
raise NotConfigured
o = cls(crawler.stats)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@ -26,8 +25,12 @@ class MemoryDebugger:
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
self.stats.set_value(
"memdebug/gc_garbage_count", len(gc.garbage), spider=spider
)
for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)
self.stats.set_value(
f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider
)

69
scrapy/extensions/memusage.py
View File

@ -3,11 +3,11 @@ MemoryUsage extension
See documentation in docs/topics/extensions.rst
"""
import sys
import socket
import logging
from pprint import pformat
import socket
import sys
from importlib import import_module
from pprint import pformat
from twisted.internet import task
@ -20,22 +20,23 @@ logger = logging.getLogger(__name__)
class MemoryUsage:
def __init__(self, crawler):
if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
raise NotConfigured
try:
# stdlib's resource module is only available on unix platforms.
self.resource = import_module('resource')
self.resource = import_module("resource")
except ImportError:
raise NotConfigured
self.crawler = crawler
self.warned = False
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
self.check_interval = crawler.settings.getfloat(
"MEMUSAGE_CHECK_INTERVAL_SECONDS"
)
self.mail = MailSender.from_settings(crawler.settings)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
@ -46,13 +47,13 @@ class MemoryUsage:
def get_virtual_size(self):
size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
if sys.platform != 'darwin':
if sys.platform != "darwin":
# on macOS ru_maxrss is in bytes, on Linux it is in KB
size *= 1024
return size
def engine_started(self):
self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())
self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
self.tasks = []
tsk = task.LoopingCall(self.update)
self.tasks.append(tsk)
@ -72,42 +73,56 @@ class MemoryUsage:
tsk.stop()
def update(self):
self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
def _check_limit(self):
if self.get_virtual_size() > self.limit:
self.crawler.stats.set_value('memusage/limit_reached', 1)
peak_mem_usage = self.get_virtual_size()
if peak_mem_usage > self.limit:
self.crawler.stats.set_value("memusage/limit_reached", 1)
mem = self.limit / 1024 / 1024
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
{'memusage': mem}, extra={'crawler': self.crawler})
logger.error(
"Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
{"memusage": mem},
extra={"crawler": self.crawler},
)
if self.notify_mails:
subj = (
f"{self.crawler.settings['BOT_NAME']} terminated: "
f"memory usage exceeded {mem}M at {socket.gethostname()}"
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/limit_notified', 1)
self.crawler.stats.set_value("memusage/limit_notified", 1)
if self.crawler.engine.spider is not None:
self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
self.crawler.engine.close_spider(
self.crawler.engine.spider, "memusage_exceeded"
)
else:
self.crawler.stop()
else:
logger.info(
"Peak memory usage is %(virtualsize)dMiB",
{"virtualsize": peak_mem_usage / 1024 / 1024},
)
def _check_warning(self):
if self.warned: # warn only once
return
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value('memusage/warning_reached', 1)
self.crawler.stats.set_value("memusage/warning_reached", 1)
mem = self.warning / 1024 / 1024
logger.warning("Memory usage reached %(memusage)dM",
{'memusage': mem}, extra={'crawler': self.crawler})
logger.warning(
"Memory usage reached %(memusage)dMiB",
{"memusage": mem},
extra={"crawler": self.crawler},
)
if self.notify_mails:
subj = (
f"{self.crawler.settings['BOT_NAME']} warning: "
f"memory usage reached {mem}M at {socket.gethostname()}"
f"memory usage reached {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/warning_notified', 1)
self.crawler.stats.set_value("memusage/warning_notified", 1)
self.warned = True
def _send_report(self, rcpts, subject):
@ -117,7 +132,9 @@ class MemoryUsage:
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += (
"ENGINE STATUS ------------------------------------------------------- \r\n"
)
s += "\r\n"
s += pformat(get_engine_status(self.crawler.engine))
s += "\r\n"

27
scrapy/extensions/postprocessing.py
View File

@ -29,8 +29,13 @@ class GzipPlugin:
compress_level = self.feed_options.get("gzip_compresslevel", 9)
mtime = self.feed_options.get("gzip_mtime")
filename = self.feed_options.get("gzip_filename")
self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,
mtime=mtime, filename=filename)
self.gzipfile = GzipFile(
fileobj=self.file,
mode="wb",
compresslevel=compress_level,
mtime=mtime,
filename=filename,
)
def write(self, data: bytes) -> int:
return self.gzipfile.write(data)
@ -55,7 +60,9 @@ class Bz2Plugin:
self.file = file
self.feed_options = feed_options
compress_level = self.feed_options.get("bz2_compresslevel", 9)
self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)
self.bz2file = BZ2File(
filename=self.file, mode="wb", compresslevel=compress_level
)
def write(self, data: bytes) -> int:
return self.bz2file.write(data)
@ -90,8 +97,14 @@ class LZMAPlugin:
check = self.feed_options.get("lzma_check", -1)
preset = self.feed_options.get("lzma_preset")
filters = self.feed_options.get("lzma_filters")
self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,
check=check, preset=preset, filters=filters)
self.lzmafile = LZMAFile(
filename=self.file,
mode="wb",
format=format,
check=check,
preset=preset,
filters=filters,
)
def write(self, data: bytes) -> int:
return self.lzmafile.write(data)
@ -114,7 +127,9 @@ class PostProcessingManager(IOBase):
:type file: file like object
"""
def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:
def __init__(
self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]
) -> None:
self.plugins = self._load_plugins(plugins)
self.file = file
self.feed_options = feed_options

12
scrapy/extensions/spiderstate.py
View File

@ -1,5 +1,5 @@
import os
import pickle
from pathlib import Path
from scrapy import signals
from scrapy.exceptions import NotConfigured
@ -25,16 +25,16 @@ class SpiderState:
def spider_closed(self, spider):
if self.jobdir:
with open(self.statefn, 'wb') as f:
with Path(self.statefn).open("wb") as f:
pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider):
if self.jobdir and os.path.exists(self.statefn):
with open(self.statefn, 'rb') as f:
if self.jobdir and Path(self.statefn).exists():
with Path(self.statefn).open("rb") as f:
spider.state = pickle.load(f)
else:
spider.state = {}
@property
def statefn(self):
return os.path.join(self.jobdir, 'spider.state')
def statefn(self) -> str:
return str(Path(self.jobdir, "spider.state"))

3
scrapy/extensions/statsmailer.py
View File

@ -5,12 +5,11 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
"""
from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
class StatsMailer:
def __init__(self, stats, recipients, mail):
self.stats = stats
self.recipients = recipients

Some files were not shown because too many files have changed in this diff Show More