RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'scrapy/master' into integrate-mime

This commit is contained in:
Adrián Chaves 2023-12-29 08:30:44 +01:00
parent 1ade9d2ed7 34e01a8a93
commit 2d9857b441
411 changed files with 26506 additions and 17300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bandit.yml
View File

@ -1,5 +1,6 @@
skips:
- B101
- B113 # https://github.com/PyCQA/bandit/issues/1010
- B105
- B301
- B303
@ -17,3 +18,4 @@ skips:
- B503
- B603
- B605
exclude_dirs: ['tests']

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.7.1
current_version = 2.11.0
commit = True
tag = True
tag_name = {new_version}

2
.flake8
View File

@ -1,7 +1,7 @@
[flake8]
max-line-length = 119
ignore = W503
ignore = W503, E203
exclude =
docs/conf.py

7
.git-blame-ignore-revs Normal file
View File

@ -0,0 +1,7 @@
# .git-blame-ignore-revs
# adding black formatter to all the code
e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
# re applying black to the code with default line length
303f0a70fcf8067adf0a909c2096a5009162383a
# reaplying black again and removing line length on pre-commit black config
c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962

13
.github/ISSUE_TEMPLATE/question.md vendored Normal file
View File

@ -0,0 +1,13 @@
---
name: Question / Help
about: Ask a question about Scrapy or ask for help with your Scrapy code.
---
Thanks for taking an interest in Scrapy!
The Scrapy GitHub issue tracker is not meant for questions or help. Please ask
for help in the [Scrapy community resources](https://scrapy.org/community/)
instead.
The GitHub issue tracker's purpose is to deal with bug reports and feature
requests for the project itself.

24
.github/workflows/checks.yml vendored
View File

@ -1,6 +1,10 @@
name: Checks
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
checks:
runs-on: ubuntu-latest
@ -8,27 +12,21 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.11"
env:
TOXENV: security
- python-version: "3.11"
env:
TOXENV: flake8
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: pylint
- python-version: 3.7
- python-version: 3.8
env:
TOXENV: typing
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
@ -40,3 +38,9 @@ jobs:
run: |
pip install -U tox
tox
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pre-commit/action@v3.0.0

44
.github/workflows/publish.yml vendored
View File

@ -1,31 +1,25 @@
name: Publish
on: [push]
on:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
publish:
runs-on: ubuntu-latest
if: startsWith(github.event.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Check Tag
id: check-release-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
echo ::set-output name=release_tag::true
fi
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade build twine
python -m build
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: 3.12
- run: |
pip install --upgrade build twine
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@v1.6.4
with:
password: ${{ secrets.PYPI_TOKEN }}

8
.github/workflows/tests-macos.yml vendored
View File

@ -1,16 +1,20 @@
name: macOS
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests:
runs-on: macos-11
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4

36
.github/workflows/tests-ubuntu.yml vendored
View File

@ -1,6 +1,10 @@
name: Ubuntu
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests:
runs-on: ubuntu-latest
@ -8,9 +12,6 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
@ -20,30 +21,45 @@ jobs:
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: asyncio
- python-version: pypy3.9
env:
TOXENV: pypy3
- python-version: pypy3.10
env:
TOXENV: pypy3
# pinned deps
- python-version: 3.7.13
- python-version: 3.8.17
env:
TOXENV: pinned
- python-version: 3.7.13
- python-version: 3.8.17
env:
TOXENV: asyncio-pinned
- python-version: pypy3.7
- python-version: pypy3.8
env:
TOXENV: pypy3-pinned
- python-version: 3.8.17
env:
TOXENV: extra-deps-pinned
- python-version: 3.8.17
env:
TOXENV: botocore-pinned
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: extra-deps
- python-version: "3.12"
env:
TOXENV: botocore
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
@ -51,7 +67,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev

26
.github/workflows/tests-windows.yml vendored
View File

@ -1,6 +1,10 @@
name: Windows
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests:
runs-on: windows-latest
@ -8,31 +12,27 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.7
env:
TOXENV: windows-pinned
- python-version: 3.8
env:
TOXENV: py
TOXENV: windows-pinned
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: asyncio
# no binary package for lxml for 3.11 yet
# - python-version: "3.11"
# env:
# TOXENV: py
# - python-version: "3.11"
# env:
# TOXENV: asyncio
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4

2
.isort.cfg Normal file
View File

@ -0,0 +1,2 @@
[settings]
profile = black

24
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,24 @@
repos:
- repo: https://github.com/PyCQA/bandit
rev: 1.7.5
hooks:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 6.1.0
hooks:
- id: flake8
- repo: https://github.com/psf/black.git
rev: 23.9.1
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.16.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==23.9.1

1
MANIFEST.in
View File

@ -7,6 +7,7 @@ include NEWS
include scrapy/VERSION
include scrapy/mime.types
include scrapy/py.typed
include codecov.yml
include conftest.py

13
README.rst
View File

@ -17,9 +17,10 @@ Scrapy
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
.. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
.. :alt: macOS
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
@ -41,7 +42,7 @@ Scrapy
Overview
========
Scrapy is a fast high-level web crawling and web scraping framework, used to
Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
@ -58,7 +59,7 @@ including a list of features.
Requirements
============
* Python 3.7+
* Python 3.8+
* Works on Linux, Windows, macOS, BSD
Install
@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list.
Commercial Support
==================
See https://scrapy.org/support/ for details.
See https://scrapy.org/support/ for details.

8
artwork/README.rst
View File

@ -2,19 +2,19 @@
Scrapy artwork
==============
This folder contains Scrapy artwork resources such as logos and fonts.
This folder contains the Scrapy artwork resources such as logos and fonts.
scrapy-logo.jpg
---------------
Main Scrapy logo, in JPEG format.
The main Scrapy logo, in JPEG format.
qlassik.zip
-----------
Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
scrapy-blog.logo.xcf
--------------------
The logo used in Scrapy blog, in Gimp format.
The logo used in the Scrapy blog, in Gimp format.

44
conftest.py
View File

@ -1,36 +1,43 @@
import platform
import sys
from pathlib import Path
import pytest
from twisted import version as twisted_version
from twisted.python.versions import Version
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
return (str(p) for p in Path(folder).rglob("*.py"))
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
"tests/ftpserver.py",
"tests/mockserver.py",
"tests/pipelines.py",
"tests/spiders.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
]
with Path('tests/ignores.txt').open(encoding="utf-8") as reader:
with Path("tests/ignores.txt").open(encoding="utf-8") as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
if file_path and file_path[0] != "#":
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
"scrapy/core/downloader/handlers/http2.py",
*_py_files("scrapy/core/http2"),
)
)
@ -50,7 +57,7 @@ def pytest_addoption(parser):
)
@pytest.fixture(scope='class')
@pytest.fixture(scope="class")
def reactor_pytest(request):
if not request.cls:
# doctests
@ -61,14 +68,31 @@ def reactor_pytest(request):
@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
pytest.skip("This test is only run with --reactor=asyncio")
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
pytest.skip('This test is only run without --reactor=asyncio')
if (
request.node.get_closest_marker("only_not_asyncio")
and reactor_pytest == "asyncio"
):
pytest.skip("This test is only run without --reactor=asyncio")
@pytest.fixture(autouse=True)
def requires_uvloop(request):
if not request.node.get_closest_marker("requires_uvloop"):
return
if sys.implementation.name == "pypy":
pytest.skip("uvloop does not support pypy properly")
if platform.system() == "Windows":
pytest.skip("uvloop does not support Windows")
if twisted_version == Version("twisted", 21, 2, 0):
pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
if sys.version_info >= (3, 12):
pytest.skip("uvloop doesn't support Python 3.12 yet")
def pytest_configure(config):

2
docs/Makefile
View File

@ -87,7 +87,7 @@ coverage: build
htmlview: html
$(PYTHON) -c "import webbrowser; from pathlib import Path; \
webbrowser.open('file://' + Path('build/html/index.html').resolve())"
webbrowser.open(Path('build/html/index.html').resolve().as_uri())"
clean:
-rm -rf build/*

79
docs/_ext/scrapydocs.py
View File

@ -1,7 +1,8 @@
from operator import itemgetter
from docutils.parsers.rst.roles import set_classes
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst.roles import set_classes
from sphinx.util.nodes import make_refnode
@ -11,15 +12,15 @@ class settingslist_node(nodes.General, nodes.Element):
class SettingsListDirective(Directive):
def run(self):
return [settingslist_node('')]
return [settingslist_node("")]
def is_setting_index(node):
if node.tagname == 'index' and node['entries']:
if node.tagname == "index" and node["entries"]:
# index entries for setting directives look like:
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
entry_type, info, refid = node["entries"][0][:3]
return entry_type == "pair" and info.endswith("; setting")
return False
@ -30,14 +31,14 @@ def get_setting_target(node):
def get_setting_name_and_refid(node):
"""Extract setting name from directive index node"""
entry_type, info, refid = node['entries'][0][:3]
return info.replace('; setting', ''), refid
entry_type, info, refid = node["entries"][0][:3]
return info.replace("; setting", ""), refid
def collect_scrapy_settings_refs(app, doctree):
env = app.builder.env
if not hasattr(env, 'scrapy_all_settings'):
if not hasattr(env, "scrapy_all_settings"):
env.scrapy_all_settings = []
for node in doctree.traverse(is_setting_index):
@ -46,18 +47,23 @@ def collect_scrapy_settings_refs(app, doctree):
setting_name, refid = get_setting_name_and_refid(node)
env.scrapy_all_settings.append({
'docname': env.docname,
'setting_name': setting_name,
'refid': refid,
})
env.scrapy_all_settings.append(
{
"docname": env.docname,
"setting_name": setting_name,
"refid": refid,
}
)
def make_setting_element(setting_data, app, fromdocname):
refnode = make_refnode(app.builder, fromdocname,
todocname=setting_data['docname'],
targetid=setting_data['refid'],
child=nodes.Text(setting_data['setting_name']))
refnode = make_refnode(
app.builder,
fromdocname,
todocname=setting_data["docname"],
targetid=setting_data["refid"],
child=nodes.Text(setting_data["setting_name"]),
)
p = nodes.paragraph()
p += refnode
@ -71,10 +77,13 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
for node in doctree.traverse(settingslist_node):
settings_list = nodes.bullet_list()
settings_list.extend([make_setting_element(d, app, fromdocname)
for d in sorted(env.scrapy_all_settings,
key=itemgetter('setting_name'))
if fromdocname != d['docname']])
settings_list.extend(
[
make_setting_element(d, app, fromdocname)
for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name"))
if fromdocname != d["docname"]
]
)
node.replace_self(settings_list)
@ -99,41 +108,41 @@ def setup(app):
rolename="reqmeta",
indextemplate="pair: %s; reqmeta",
)
app.add_role('source', source_role)
app.add_role('commit', commit_role)
app.add_role('issue', issue_role)
app.add_role('rev', rev_role)
app.add_role("source", source_role)
app.add_role("commit", commit_role)
app.add_role("issue", issue_role)
app.add_role("rev", rev_role)
app.add_node(settingslist_node)
app.add_directive('settingslist', SettingsListDirective)
app.add_directive("settingslist", SettingsListDirective)
app.connect('doctree-read', collect_scrapy_settings_refs)
app.connect('doctree-resolved', replace_settingslist_nodes)
app.connect("doctree-read", collect_scrapy_settings_refs)
app.connect("doctree-resolved", replace_settingslist_nodes)
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
ref = "https://github.com/scrapy/scrapy/blob/master/" + text
set_classes(options)
node = nodes.reference(rawtext, text, refuri=ref, **options)
return [node], []
def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/issues/' + text
ref = "https://github.com/scrapy/scrapy/issues/" + text
set_classes(options)
node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options)
return [node], []
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/commit/' + text
ref = "https://github.com/scrapy/scrapy/commit/" + text
set_classes(options)
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options)
return [node], []
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
ref = "http://hg.scrapy.org/scrapy/changeset/" + text
set_classes(options)
node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "r" + text, refuri=ref, **options)
return [node], []

11
docs/_templates/layout.html vendored
View File

@ -1,11 +0,0 @@
{% extends "!layout.html" %}
{% block footer %}
{{ super() }}
<script type="text/javascript">
analytics.ready(function () {
ga('require', 'linker');
ga('linker:autoLink', ['zyte.com']);
});
</script>
{% endblock %}

2
docs/_tests/quotes.html
View File

@ -273,7 +273,7 @@
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
</p>
</div>
</footer>

2
docs/_tests/quotes1.html
View File

@ -273,7 +273,7 @@
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
</p>
</div>
</footer>

175
docs/conf.py
View File

@ -25,30 +25,30 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'hoverxref.extension',
'notfound.extension',
'scrapydocs',
'sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
"hoverxref.extension",
"notfound.extension",
"scrapydocs",
"sphinx.ext.autodoc",
"sphinx.ext.coverage",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8'
# source_encoding = 'utf-8'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'Scrapy'
copyright = f'2008–{datetime.now().year}, Scrapy developers'
project = "Scrapy"
copyright = f"2008–{datetime.now().year}, Scrapy developers"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -57,50 +57,51 @@ copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The short X.Y version.
try:
import scrapy
version = '.'.join(map(str, scrapy.version_info[:2]))
version = ".".join(map(str, scrapy.version_info[:2]))
release = scrapy.__version__
except ImportError:
version = ''
release = ''
version = ""
release = ""
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
language = 'en'
language = "en"
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of documents that shouldn't be included in the build.
#unused_docs = []
# unused_docs = []
exclude_patterns = ['build']
exclude_patterns = ["build"]
# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['.build']
exclude_trees = [".build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# List of Sphinx warnings that will not be raised
suppress_warnings = ['epub.unknown_project_files']
suppress_warnings = ["epub.unknown_project_files"]
# Options for HTML output
@ -108,17 +109,18 @@ suppress_warnings = ['epub.unknown_project_files']
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
# Add path to the RTD explicitly to robustify builds (otherwise might
# fail in a clean Debian build env)
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
@ -128,44 +130,44 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'
html_last_updated_fmt = "%b %d, %Y"
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_use_modindex = True
# html_use_modindex = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, the reST sources are included in the HTML build as _sources/<name>.
html_copy_source = True
@ -173,16 +175,16 @@ html_copy_source = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''
# html_file_suffix = ''
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
htmlhelp_basename = "Scrapydoc"
html_css_files = [
'custom.css',
"custom.css",
]
@ -190,34 +192,33 @@ html_css_files = [
# ------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', 'Scrapy Documentation',
'Scrapy developers', 'manual'),
("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_use_modindex = True
# latex_use_modindex = True
# Options for the linkcheck builder
@ -226,8 +227,9 @@ latex_documents = [
# A list of regular expressions that match URIs that should not be checked when
# doing a linkcheck build.
linkcheck_ignore = [
'http://localhost:\d+', 'http://hg.scrapy.org',
'http://directory.google.com/'
"http://localhost:\d+",
"http://hg.scrapy.org",
"http://directory.google.com/",
]
@ -237,44 +239,35 @@ coverage_ignore_pyobjects = [
# Contract’s add_pre_hook and add_post_hook are not documented because
# they should be transparent to contract developers, for whom pre_hook and
# post_hook should be the actual concern.
r'\bContract\.add_(pre|post)_hook$',
r"\bContract\.add_(pre|post)_hook$",
# ContractsManager is an internal class, developers are not expected to
# interact with it directly in any way.
r'\bContractsManager\b$',
r"\bContractsManager\b$",
# For default contracts we only want to document their general purpose in
# their __init__ method, the methods they reimplement to achieve that purpose
# should be irrelevant to developers using those contracts.
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
r"\w+Contract\.(adjust_request_args|(pre|post)_process)$",
# Methods of downloader middlewares are not documented, only the classes
# themselves, since downloader middlewares are controlled through Scrapy
# settings.
r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
r"^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.",
# Base classes of downloader middlewares are implementation details that
# are not meant for users.
r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware",
# Private exception used by the command-line interface implementation.
r'^scrapy\.exceptions\.UsageError',
r"^scrapy\.exceptions\.UsageError",
# Methods of BaseItemExporter subclasses are only documented in
# BaseItemExporter.
r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
r"^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.",
# Extension behavior is only modified through settings. Methods of
# extension classes, as well as helper functions, are implementation
# details that are not documented.
r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
r"^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.", # methods
r"^scrapy\.extensions\.[a-z]\w*?\.[a-z]", # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
r"^scrapy\.linkextractors\.FilteringLinkExtractor$",
# Implementation detail of LxmlLinkExtractor
r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
]
@ -282,18 +275,18 @@ coverage_ignore_pyobjects = [
# -------------------------------------
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cryptography' : ('https://cryptography.io/en/latest/', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.wiki/en/latest/', None),
'twisted': ('https://docs.twisted.org/en/stable/', None),
'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
"attrs": ("https://www.attrs.org/en/stable/", None),
"coverage": ("https://coverage.readthedocs.io/en/latest", None),
"cryptography": ("https://cryptography.io/en/latest/", None),
"cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
"itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
"pytest": ("https://docs.pytest.org/en/latest", None),
"python": ("https://docs.python.org/3", None),
"sphinx": ("https://www.sphinx-doc.org/en/master", None),
"tox": ("https://tox.wiki/en/latest/", None),
"twisted": ("https://docs.twisted.org/en/stable/", None),
"twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
"w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
}
intersphinx_disabled_reftypes = []
@ -313,16 +306,16 @@ hoverxref_role_types = {
"setting": "tooltip",
"signal": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
def setup(app):
app.connect('autodoc-skip-member', maybe_skip_member)
app.connect("autodoc-skip-member", maybe_skip_member)
def maybe_skip_member(app, what, name, obj, skip, options):
if not skip:
# autodocs was generating a text "alias of" for the following members
# https://github.com/sphinx-doc/sphinx/issues/4422
return name in {'default_item_class', 'default_selector_class'}
return name in {"default_item_class", "default_selector_class"}
return skip

8
docs/conftest.py
View File

@ -15,20 +15,20 @@ from scrapy.http.response.html import HtmlResponse
def load_response(url: str, filename: str) -> HtmlResponse:
input_path = Path(__file__).parent / '_tests' / filename
input_path = Path(__file__).parent / "_tests" / filename
return HtmlResponse(url, body=input_path.read_bytes())
def setup(namespace):
namespace['load_response'] = load_response
namespace["load_response"] = load_response
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
PythonCodeBlockParser(future_imports=['print_function']),
PythonCodeBlockParser(future_imports=["print_function"]),
skip,
],
pattern='*.rst',
pattern="*.rst",
setup=setup,
).pytest()

66
docs/contributing.rst
View File

@ -11,10 +11,6 @@ Contributing to Scrapy
There are many ways to contribute to Scrapy. Here are some of them:
* Blog about Scrapy. Tell the world how you're using Scrapy. This will help
newcomers with more examples and will help the Scrapy project to increase its
visibility.
* Report bugs and request features in the `issue tracker`_, trying to follow
the guidelines detailed in `Reporting bugs`_ below.
@ -22,13 +18,16 @@ There are many ways to contribute to Scrapy. Here are some of them:
:ref:`writing-patches` and `Submitting patches`_ below for details on how to
write and submit a patch.
* Blog about Scrapy. Tell the world how you're using Scrapy. This will help
newcomers with more examples and will help the Scrapy project to increase its
visibility.
* Join the `Scrapy subreddit`_ and share your ideas on how to
improve Scrapy. We're always open to suggestions.
* Answer Scrapy questions at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__.
Reporting bugs
==============
@ -49,7 +48,7 @@ guidelines when you're going to report a new bug.
(use "scrapy" tag).
* check the `open issues`_ to see if the issue has already been reported. If it
has, don't dismiss the report, but check the ticket history and comments. If
has, don't dismiss the report, but check the ticket history and comments. If
you have additional useful information, please leave a comment, or consider
:ref:`sending a pull request <writing-patches>` with a fix.
@ -80,6 +79,13 @@ guidelines when you're going to report a new bug.
Writing patches
===============
Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you
can work on. These issues are a great way to get started with contributing to
Scrapy. If you're new to the codebase, you may want to focus on documentation
or testing-related issues, as they are always useful and can help you get
more familiar with the project. You can also check Scrapy's `test coverage`_
to see which areas may benefit from more tests.
The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged.
Well-written patches should:
@ -169,16 +175,43 @@ Coding style
Please follow these coding conventions when writing code for inclusion in
Scrapy:
* Unless otherwise specified, follow :pep:`8`.
* It's OK to use lines longer than 79 chars if it improves the code
readability.
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
run black manually with ``tox -e black``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
.. _scrapy-pre-commit:
Pre-commit
==========
We use `pre-commit`_ to automatically address simple code issues before every
commit.
.. _pre-commit: https://pre-commit.com/
After your create a local clone of your fork of the Scrapy repository:
#. `Install pre-commit <https://pre-commit.com/#installation>`_.
#. On the root of your local clone of the Scrapy repository, run the following
command:
.. code-block:: bash
pre-commit install
Now pre-commit will check your changes every time you create a Git commit. Upon
finding issues, pre-commit aborts your commit, and either fixes those issues
automatically, or only reports them to you. If it fixes those issues
automatically, creating your commit again should succeed. Otherwise, you may
need to address the corresponding issues manually first.
.. _documentation-policies:
Documentation policies
@ -232,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
the tests with Python 3.7 use::
the tests with Python 3.10 use::
tox -e py37
tox -e py310
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py37,py38 -p auto
tox -e py39,py310 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@ -250,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
tox -e py37 -- scrapy tests -n auto
tox -e py310 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:
@ -287,3 +320,6 @@ And their unit-tests are in::
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy

55
docs/faq.rst
View File

@ -35,8 +35,10 @@ for parsing HTML responses in Scrapy callbacks.
You just have to feed the response's body into a ``BeautifulSoup`` object
and extract whatever data you need from it.
Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser::
Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:
.. skip: next
.. code-block:: python
from bs4 import BeautifulSoup
import scrapy
@ -45,17 +47,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
class ExampleSpider(scrapy.Spider):
name = "example"
allowed_domains = ["example.com"]
start_urls = (
'http://www.example.com/',
)
start_urls = ("http://www.example.com/",)
def parse(self, response):
# use lxml to get decent HTML parsing speed
soup = BeautifulSoup(response.text, 'lxml')
yield {
"url": response.url,
"title": soup.h1.string
}
soup = BeautifulSoup(response.text, "lxml")
yield {"url": response.url, "title": soup.h1.string}
.. note::
@ -109,11 +106,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient
in most cases.
If you do want to crawl in true `BFO order`_, you can do it by
setting the following settings::
setting the following settings:
.. code-block:: python
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
While pending requests are below the configured values of
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
@ -159,11 +158,13 @@ See also other suggestions at `StackOverflow`_.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation::
your custom implementation:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
'myproject.middlewares.CustomOffsiteMiddleware': 500,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
"myproject.middlewares.CustomOffsiteMiddleware": 500,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
@ -230,16 +231,20 @@ Can I return (Twisted) deferreds from signal handlers?
Some signals support returning deferreds from their handlers, others don't. See
the :ref:`topics-signals-ref` to know which ones.
What does the response status code 999 means?
---------------------------------------------
What does the response status code 999 mean?
--------------------------------------------
999 is a custom response status code used by Yahoo sites to throttle requests.
Try slowing down the crawling speed by using a download delay of ``2`` (or
higher) in your spider::
higher) in your spider:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class MySpider(CrawlSpider):
name = 'myspider'
name = "myspider"
download_delay = 2
@ -351,19 +356,21 @@ How to split an item into multiple items in an item pipeline?
input item. :ref:`Create a spider middleware <custom-spider-middleware>`
instead, and use its
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method for this purpose. For example::
method for this purpose. For example:
.. code-block:: python
from copy import deepcopy
from itemadapter import is_item, ItemAdapter
class MultiplyItemsMiddleware:
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if is_item(item):
adapter = ItemAdapter(item)
for _ in range(adapter['multiply_by']):
for _ in range(adapter["multiply_by"]):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?
@ -413,4 +420,4 @@ See :issue:`2680`.
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search

4
docs/index.rst
View File

@ -222,6 +222,7 @@ Extending Scrapy
:hidden:
topics/architecture
topics/addons
topics/downloader-middleware
topics/spider-middleware
topics/extensions
@ -235,6 +236,9 @@ Extending Scrapy
:doc:`topics/architecture`
Understand the Scrapy architecture.
:doc:`topics/addons`
Enable and configure third-party extensions.
:doc:`topics/downloader-middleware`
Customize how pages get requested and downloaded.

2
docs/intro/install.rst
View File

@ -9,7 +9,7 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.7+, either the CPython implementation (default) or
Scrapy requires Python 3.8+, either the CPython implementation (default) or
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:

14
docs/intro/overview.rst
View File

@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
example of a Scrapy Spider using the simplest way to run a spider.
Here's the code for a spider that scrapes famous quotes from website
https://quotes.toscrape.com, following the pagination::
https://quotes.toscrape.com, following the pagination:
.. code-block:: python
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/tag/humor/',
"https://quotes.toscrape.com/tag/humor/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'author': quote.xpath('span/small/text()').get(),
'text': quote.css('span.text::text').get(),
"author": quote.xpath("span/small/text()").get(),
"text": quote.css("span.text::text").get(),
}
next_page = response.css('li.next a::attr("href")').get()

283
docs/intro/tutorial.rst
View File

@ -83,7 +83,9 @@ optionally how to follow links in the pages, and how to parse the downloaded
page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:
.. code-block:: python
from pathlib import Path
@ -95,17 +97,17 @@ This is the code for our first Spider. Save it in a file named
def start_requests(self):
urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
filename = f"quotes-{page}.html"
Path(filename).write_bytes(response.body)
self.log(f'Saved file {filename}')
self.log(f"Saved file {filename}")
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
@ -177,7 +179,9 @@ that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
for your spider.
.. code-block:: python
from pathlib import Path
@ -187,13 +191,13 @@ for your spider::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
filename = f"quotes-{page}.html"
Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
@ -245,8 +249,10 @@ object:
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
.. code-block:: pycon
>>> response.css("title")
[<Selector query='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@ -256,42 +262,54 @@ data.
To extract the text from the title above, you can do:
>>> response.css('title::text').getall()
['Quotes to Scrape']
.. code-block:: pycon
>>> response.css("title::text").getall()
['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
CSS query, to mean we want to select only the text elements directly inside
``<title>`` element. If we don't specify ``::text``, we'd get the full title
element, including its tags:
>>> response.css('title').getall()
['<title>Quotes to Scrape</title>']
.. code-block:: pycon
>>> response.css("title").getall()
['<title>Quotes to Scrape</title>']
The other thing is that the result of calling ``.getall()`` is a list: it is
possible that a selector returns more than one result, so we extract them all.
When you know you just want the first result, as in this case, you can do:
>>> response.css('title::text').get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.css("title::text").get()
'Quotes to Scrape'
As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.css("title::text")[0].get()
'Quotes to Scrape'
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
raise an :exc:`IndexError` exception if there are no results::
raise an :exc:`IndexError` exception if there are no results:
>>> response.css('noelement')[0].get()
.. code-block:: pycon
>>> response.css("noelement")[0].get()
Traceback (most recent call last):
...
IndexError: list index out of range
You might want to use ``.get()`` directly on the
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
if there are no results::
if there are no results:
>>> response.css("noelement").get()
.. code-block:: pycon
>>> response.css("noelement").get()
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
@ -302,14 +320,16 @@ Besides the :meth:`~scrapy.selector.SelectorList.getall` and
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
:doc:`regular expressions <library/re>`:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
.. code-block:: pycon
In order to find the proper CSS selectors to use, you might find useful opening
>>> response.css("title::text").re(r"Quotes.*")
['Quotes to Scrape']
>>> response.css("title::text").re(r"Q\w+")
['Quotes']
>>> response.css("title::text").re(r"(\w+) to (\w+)")
['Quotes', 'Scrape']
In order to find the proper CSS selectors to use, you might find it useful to open
the response page from the shell in your web browser using ``view(response)``.
You can use your browser's developer tools to inspect the HTML and come up
with a selector (see :ref:`topics-developer-tools`).
@ -325,10 +345,12 @@ XPath: a brief intro
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.xpath("//title")
[<Selector query='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath("//title/text()").get()
'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@ -385,33 +407,41 @@ we want::
We get a list of selectors for the quote HTML elements with:
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
.. code-block:: pycon
>>> response.css("div.quote")
[<Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
variable, so that we can run our CSS selectors directly on a particular quote:
>>> quote = response.css("div.quote")[0]
.. code-block:: pycon
>>> quote = response.css("div.quote")[0]
Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created:
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
.. code-block:: pycon
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
Given that the tags are a list of strings, we can use the ``.getall()`` method
to get all of them:
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. code-block:: pycon
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. invisible-code-block: python
@ -420,14 +450,17 @@ to get all of them:
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
.. code-block:: pycon
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
...
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
Extracting data in our spider
-----------------------------
@ -438,7 +471,9 @@ extraction logic above into our spider.
A Scrapy spider typically generates many dictionaries containing the data
extracted from the page. To do that, we use the ``yield`` Python keyword
in the callback, as you can see below::
in the callback, as you can see below:
.. code-block:: python
import scrapy
@ -446,19 +481,27 @@ in the callback, as you can see below::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
If you run this spider, it will output the extracted data with the log::
To run this spider, exit the scrapy shell by entering::
quit()
Then, run::
scrapy crawl quotes
Now, it should output the extracted data with the log::
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
@ -533,17 +576,23 @@ This gets the anchor element, but we want the attribute ``href``. For that,
Scrapy supports a CSS extension that lets you select the attribute contents,
like this:
>>> response.css('li.next a::attr(href)').get()
'/page/2/'
.. code-block:: pycon
>>> response.css("li.next a::attr(href)").get()
'/page/2/'
There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more):
>>> response.css('li.next a').attrib['href']
'/page/2/'
.. code-block:: pycon
>>> response.css("li.next a").attrib["href"]
'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
page, extracting data from it:
.. code-block:: python
import scrapy
@ -551,18 +600,18 @@ page, extracting data from it::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
"https://quotes.toscrape.com/page/1/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
@ -594,7 +643,9 @@ A shortcut for creating Requests
--------------------------------
As a shortcut for creating Request objects you can use
:meth:`response.follow <scrapy.http.TextResponse.follow>`::
:meth:`response.follow <scrapy.http.TextResponse.follow>`:
.. code-block:: python
import scrapy
@ -602,18 +653,18 @@ As a shortcut for creating Request objects you can use
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
"https://quotes.toscrape.com/page/1/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('span small::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("span small::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
@ -621,58 +672,72 @@ Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no
need to call urljoin. Note that ``response.follow`` just returns a Request
instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
.. skip: start
for href in response.css('ul.pager a::attr(href)'):
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes:
.. code-block:: python
for href in response.css("ul.pager a::attr(href)"):
yield response.follow(href, callback=self.parse)
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
attribute automatically. So the code can be shortened further:
for a in response.css('ul.pager a'):
.. code-block:: python
for a in response.css("ul.pager a"):
yield response.follow(a, callback=self.parse)
To create multiple requests from an iterable, you can use
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead:
anchors = response.css('ul.pager a')
.. code-block:: python
anchors = response.css("ul.pager a")
yield from response.follow_all(anchors, callback=self.parse)
or, shortening it further::
or, shortening it further:
yield from response.follow_all(css='ul.pager a', callback=self.parse)
.. code-block:: python
yield from response.follow_all(css="ul.pager a", callback=self.parse)
.. skip: end
More examples and patterns
--------------------------
Here is another spider that illustrates callbacks and following links,
this time for scraping author information::
this time for scraping author information:
.. code-block:: python
import scrapy
class AuthorSpider(scrapy.Spider):
name = 'author'
name = "author"
start_urls = ['https://quotes.toscrape.com/']
start_urls = ["https://quotes.toscrape.com/"]
def parse(self, response):
author_page_links = response.css('.author + a')
author_page_links = response.css(".author + a")
yield from response.follow_all(author_page_links, self.parse_author)
pagination_links = response.css('li.next a')
pagination_links = response.css("li.next a")
yield from response.follow_all(pagination_links, self.parse)
def parse_author(self, response):
def extract_with_css(query):
return response.css(query).get(default='').strip()
return response.css(query).get(default="").strip()
yield {
'name': extract_with_css('h3.author-title::text'),
'birthdate': extract_with_css('.author-born-date::text'),
'bio': extract_with_css('.author-description::text'),
"name": extract_with_css("h3.author-title::text"),
"birthdate": extract_with_css(".author-born-date::text"),
"bio": extract_with_css(".author-description::text"),
}
This spider will start from the main page, it will follow all the links to the
@ -720,7 +785,9 @@ spider attributes by default.
In this example, the value provided for the ``tag`` argument will be available
via ``self.tag``. You can use this to make your spider fetch only quotes
with a specific tag, building the URL based on the argument::
with a specific tag, building the URL based on the argument:
.. code-block:: python
import scrapy
@ -729,20 +796,20 @@ with a specific tag, building the URL based on the argument::
name = "quotes"
def start_requests(self):
url = 'https://quotes.toscrape.com/'
tag = getattr(self, 'tag', None)
url = "https://quotes.toscrape.com/"
tag = getattr(self, "tag", None)
if tag is not None:
url = url + 'tag/' + tag
url = url + "tag/" + tag
yield scrapy.Request(url, self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
yield response.follow(next_page, self.parse)

667
docs/news.rst
View File

@ -3,6 +3,639 @@
Release notes
=============
.. _release-2.11.0:
Scrapy 2.11.0 (2023-09-18)
--------------------------
Highlights:
- Spiders can now modify :ref:`settings <topics-settings>` in their
:meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider
arguments <spiderargs>`.
- Periodic logging of stats.
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Most of the initialization of :class:`scrapy.crawler.Crawler` instances is
now done in :meth:`~scrapy.crawler.Crawler.crawl`, so the state of
instances before that method is called is now different compared to older
Scrapy versions. We do not recommend using the
:class:`~scrapy.crawler.Crawler` instances before
:meth:`~scrapy.crawler.Crawler.crawl` is called. (:issue:`6038`)
- :meth:`scrapy.Spider.from_crawler` is now called before the initialization
of various components previously initialized in
:meth:`scrapy.crawler.Crawler.__init__` and before the settings are
finalized and frozen. This change was needed to allow changing the settings
in :meth:`scrapy.Spider.from_crawler`. If you want to access the final
setting values and the initialized :class:`~scrapy.crawler.Crawler`
attributes in the spider code as early as possible you can do this in
:meth:`~scrapy.Spider.start_requests` or in a handler of the
:signal:`engine_started` signal. (:issue:`6038`)
- The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now
requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or
UTF-32). If you need to deal with JSON documents in an invalid encoding,
use ``json.loads(response.text)`` instead. (:issue:`6016`)
- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by
default but it no longer does. (:issue:`6006`, :issue:`6007`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- Removed the binary export mode of
:class:`~scrapy.exporters.PythonItemExporter`, deprecated in Scrapy 1.1.0.
(:issue:`6006`, :issue:`6007`)
.. note:: If you are using this Scrapy version on Scrapy Cloud with a stack
that includes an older Scrapy version and get a "TypeError:
Unexpected options: binary" error, you may need to add
``scrapinghub-entrypoint-scrapy >= 0.14.1`` to your project
requirements or switch to a stack that includes Scrapy 2.11.
- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy
1.0.0, use :attr:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` instead. (:issue:`6010`)
Deprecations
~~~~~~~~~~~~
- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same
:class:`scrapy.crawler.Crawler` instance is now deprecated. (:issue:`1587`,
:issue:`6040`)
New features
~~~~~~~~~~~~
- Spiders can now modify settings in their
:meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider
arguments <spiderargs>`. (:issue:`1305`, :issue:`1580`, :issue:`2392`,
:issue:`3663`, :issue:`6038`)
- Added the :class:`~scrapy.extensions.periodic_log.PeriodicLog` extension
which can be enabled to log stats and/or their differences periodically.
(:issue:`5926`)
- Optimized the memory usage in :meth:`TextResponse.json
<scrapy.http.TextResponse.json>` by removing unnecessary body decoding.
(:issue:`5968`, :issue:`6016`)
- Links to ``.webp`` files are now ignored by :ref:`link extractors
<topics-link-extractors>`. (:issue:`6021`)
Bug fixes
~~~~~~~~~
- Fixed logging enabled add-ons. (:issue:`6036`)
- Fixed :class:`~scrapy.mail.MailSender` producing invalid message bodies
when the ``charset`` argument is passed to
:meth:`~scrapy.mail.MailSender.send`. (:issue:`5096`, :issue:`5118`)
- Fixed an exception when accessing ``self.EXCEPTIONS_TO_RETRY`` from a
subclass of :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`.
(:issue:`6049`, :issue:`6050`)
- :meth:`scrapy.settings.BaseSettings.getdictorlist`, used to parse
:setting:`FEED_EXPORT_FIELDS`, now handles tuple values. (:issue:`6011`,
:issue:`6013`)
- Calls to ``datetime.utcnow()``, no longer recommended to be used, have been
replaced with calls to ``datetime.now()`` with a timezone. (:issue:`6014`)
Documentation
~~~~~~~~~~~~~
- Updated a deprecated function call in a pipeline example. (:issue:`6008`,
:issue:`6009`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Extended typing hints. (:issue:`6003`, :issue:`6005`, :issue:`6031`,
:issue:`6034`)
- Pinned brotli_ to 1.0.9 for the PyPy tests as 1.1.0 breaks them.
(:issue:`6044`, :issue:`6045`)
- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`,
:issue:`6046`)
.. _release-2.10.1:
Scrapy 2.10.1 (2023-08-30)
--------------------------
Marked ``Twisted >= 23.8.0`` as unsupported. (:issue:`6024`, :issue:`6026`)
.. _release-2.10.0:
Scrapy 2.10.0 (2023-08-04)
--------------------------
Highlights:
- Added Python 3.12 support, dropped Python 3.7 support.
- The new add-ons framework simplifies configuring 3rd-party components that
support it.
- Exceptions to retry can now be configured.
- Many fixes and improvements for feed exports.
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
- Dropped support for Python 3.7. (:issue:`5953`)
- Added support for the upcoming Python 3.12. (:issue:`5984`)
- Minimum versions increased for these dependencies:
- lxml_: 4.3.0 → 4.4.1
- cryptography_: 3.4.6 → 36.0.0
- ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`)
- boto3_ is now recommended instead of botocore_ for exporting to S3.
(:issue:`5833`).
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True``
instead of ``False``. In earlier Scrapy versions empty files were created
even when this setting was ``False`` (which was a bug that is now fixed),
so the new default should keep the old behavior. (:issue:`872`,
:issue:`5847`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
returning ``None`` or modifying the ``params`` input parameter, deprecated
in Scrapy 2.6, is no longer supported. (:issue:`5994`, :issue:`5996`)
- The ``scrapy.utils.reqser`` module, deprecated in Scrapy 2.6, is removed.
(:issue:`5994`, :issue:`5996`)
- The ``scrapy.squeues`` classes ``PickleFifoDiskQueueNonRequest``,
``PickleLifoDiskQueueNonRequest``, ``MarshalFifoDiskQueueNonRequest``,
and ``MarshalLifoDiskQueueNonRequest``, deprecated in
Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5996`)
- The property ``open_spiders`` and the methods ``has_capacity`` and
``schedule`` of :class:`scrapy.core.engine.ExecutionEngine`,
deprecated in Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5998`)
- Passing a ``spider`` argument to the
:meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`,
:meth:`~scrapy.core.engine.ExecutionEngine.crawl` and
:meth:`~scrapy.core.engine.ExecutionEngine.download` methods of
:class:`scrapy.core.engine.ExecutionEngine`, deprecated in Scrapy 2.6, is
no longer supported. (:issue:`5994`, :issue:`5998`)
Deprecations
~~~~~~~~~~~~
- :class:`scrapy.utils.datatypes.CaselessDict` is deprecated, use
:class:`scrapy.utils.datatypes.CaseInsensitiveDict` instead.
(:issue:`5146`)
- Passing the ``custom`` argument to
:func:`scrapy.utils.conf.build_component_list` is deprecated, it was used
in the past to merge ``FOO`` and ``FOO_BASE`` setting values but now Scrapy
uses :func:`scrapy.settings.BaseSettings.getwithbase` to do the same.
Code that uses this argument and cannot be switched to ``getwithbase()``
can be switched to merging the values explicitly. (:issue:`5726`,
:issue:`5923`)
New features
~~~~~~~~~~~~
- Added support for :ref:`Scrapy add-ons <topics-addons>`. (:issue:`5950`)
- Added the :setting:`RETRY_EXCEPTIONS` setting that configures which
exceptions will be retried by
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`.
(:issue:`2701`, :issue:`5929`)
- Added the possiiblity to close the spider if no items were produced in the
specified time, configured by :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`.
(:issue:`5979`)
- Added support for the :setting:`AWS_REGION_NAME` setting to feed exports.
(:issue:`5980`)
- Added support for using :class:`pathlib.Path` objects that refer to
absolute Windows paths in the :setting:`FEEDS` setting. (:issue:`5939`)
Bug fixes
~~~~~~~~~
- Fixed creating empty feeds even with ``FEED_STORE_EMPTY=False``.
(:issue:`872`, :issue:`5847`)
- Fixed using absolute Windows paths when specifying output files.
(:issue:`5969`, :issue:`5971`)
- Fixed problems with uploading large files to S3 by switching to multipart
uploads (requires boto3_). (:issue:`960`, :issue:`5735`, :issue:`5833`)
- Fixed the JSON exporter writing extra commas when some exceptions occur.
(:issue:`3090`, :issue:`5952`)
- Fixed the "read of closed file" error in the CSV exporter. (:issue:`5043`,
:issue:`5705`)
- Fixed an error when a component added by the class object throws
:exc:`~scrapy.exceptions.NotConfigured` with a message. (:issue:`5950`,
:issue:`5992`)
- Added the missing :meth:`scrapy.settings.BaseSettings.pop` method.
(:issue:`5959`, :issue:`5960`, :issue:`5963`)
- Added :class:`~scrapy.utils.datatypes.CaseInsensitiveDict` as a replacement
for :class:`~scrapy.utils.datatypes.CaselessDict` that fixes some API
inconsistencies. (:issue:`5146`)
Documentation
~~~~~~~~~~~~~
- Documented :meth:`scrapy.Spider.update_settings`. (:issue:`5745`,
:issue:`5846`)
- Documented possible problems with early Twisted reactor installation and
their solutions. (:issue:`5981`, :issue:`6000`)
- Added examples of making additional requests in callbacks. (:issue:`5927`)
- Improved the feed export docs. (:issue:`5579`, :issue:`5931`)
- Clarified the docs about request objects on redirection. (:issue:`5707`,
:issue:`5937`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Added support for running tests against the installed Scrapy version.
(:issue:`4914`, :issue:`5949`)
- Extended typing hints. (:issue:`5925`, :issue:`5977`)
- Fixed the ``test_utils_asyncio.AsyncioTest.test_set_asyncio_event_loop``
test. (:issue:`5951`)
- Fixed the ``test_feedexport.BatchDeliveriesTest.test_batch_path_differ``
test on Windows. (:issue:`5847`)
- Enabled CI runs for Python 3.11 on Windows. (:issue:`5999`)
- Simplified skipping tests that depend on ``uvloop``. (:issue:`5984`)
- Fixed the ``extra-deps-pinned`` tox env. (:issue:`5948`)
- Implemented cleanups. (:issue:`5965`, :issue:`5986`)
.. _release-2.9.0:
Scrapy 2.9.0 (2023-05-08)
-------------------------
Highlights:
- Per-domain download settings.
- Compatibility with new cryptography_ and new parsel_.
- JMESPath selectors from the new parsel_.
- Bug fixes.
Deprecations
~~~~~~~~~~~~
- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to
:class:`scrapy.extensions.feedexport.FeedSlot` and the old name is
deprecated. (:issue:`5876`)
New features
~~~~~~~~~~~~
- Settings corresponding to :setting:`DOWNLOAD_DELAY`,
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
:setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors
available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`)
- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed`
signals. (:issue:`5876`)
- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a
curl command from a :class:`~scrapy.Request` object. (:issue:`5892`)
- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be
:class:`pathlib.Path` instances. (:issue:`5801`)
Bug fixes
~~~~~~~~~
- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`)
- Fixed an error when using feed postprocessing with S3 storage.
(:issue:`5500`, :issue:`5581`)
- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method.
(:issue:`5811`, :issue:`5821`)
- Fixed an error when using cryptography_ 40.0.0+ and
:setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled.
(:issue:`5857`, :issue:`5858`)
- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline`
for files on Google Cloud Storage are no longer Base64-encoded.
(:issue:`5874`, :issue:`5891`)
- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed
string values for the curl ``--data-raw`` argument, which are produced by
browsers for data that includes certain symbols. (:issue:`5899`,
:issue:`5901`)
- The :command:`parse` command now also works with async generator callbacks.
(:issue:`5819`, :issue:`5824`)
- The :command:`genspider` command now properly works with HTTPS URLs.
(:issue:`3553`, :issue:`5808`)
- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
now skips certain malformed URLs instead of raising an exception.
(:issue:`5881`)
- :func:`scrapy.utils.python.get_func_args` now supports more types of
callables. (:issue:`5872`, :issue:`5885`)
- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers.
(:issue:`5914`, :issue:`5917`)
- Fixed an error breaking user handling of send failures in
:meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
Documentation
~~~~~~~~~~~~~
- Expanded contributing docs. (:issue:`5109`, :issue:`5851`)
- Added blacken-docs_ to pre-commit and reformatted the docs with it.
(:issue:`5813`, :issue:`5816`)
- Fixed a JS issue. (:issue:`5875`, :issue:`5877`)
- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`)
- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`,
:issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`)
- Tests for most of the examples in the docs are now run as a part of CI,
found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`)
- Removed usage of deprecated Python classes. (:issue:`5849`)
- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`)
- Fixed a random failure of the ``test_feedexport.test_batch_path_differ``
test. (:issue:`5855`, :issue:`5898`)
- Updated docstrings to match output produced by parsel_ 1.8.1 so that they
don't cause test failures. (:issue:`5902`, :issue:`5919`)
- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`,
:issue:`5908`)
.. _blacken-docs: https://github.com/adamchainz/blacken-docs
.. _release-2.8.0:
Scrapy 2.8.0 (2023-02-02)
-------------------------
This is a maintenance release, with minor features, bug fixes, and cleanups.
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- The ``scrapy.utils.gz.read1`` function, deprecated in Scrapy 2.0, has now
been removed. Use the :meth:`~io.BufferedIOBase.read1` method of
:class:`~gzip.GzipFile` instead.
(:issue:`5719`)
- The ``scrapy.utils.python.to_native_str`` function, deprecated in Scrapy
2.0, has now been removed. Use :func:`scrapy.utils.python.to_unicode`
instead.
(:issue:`5719`)
- The ``scrapy.utils.python.MutableChain.next`` method, deprecated in Scrapy
2.0, has now been removed. Use
:meth:`~scrapy.utils.python.MutableChain.__next__` instead.
(:issue:`5719`)
- The ``scrapy.linkextractors.FilteringLinkExtractor`` class, deprecated
in Scrapy 2.0, has now been removed. Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead.
(:issue:`5720`)
- Support for using environment variables prefixed with ``SCRAPY_`` to
override settings, deprecated in Scrapy 2.0, has now been removed.
(:issue:`5724`)
- Support for the ``noconnect`` query string argument in proxy URLs,
deprecated in Scrapy 2.0, has now been removed. We expect proxies that used
to need it to work fine without it.
(:issue:`5731`)
- The ``scrapy.utils.python.retry_on_eintr`` function, deprecated in Scrapy
2.3, has now been removed.
(:issue:`5719`)
- The ``scrapy.utils.python.WeakKeyCache`` class, deprecated in Scrapy 2.4,
has now been removed.
(:issue:`5719`)
- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4,
has now been removed.
(:issue:`5719`)
Deprecations
~~~~~~~~~~~~
- :exc:`scrapy.pipelines.images.NoimagesDrop` is now deprecated.
(:issue:`5368`, :issue:`5489`)
- :meth:`ImagesPipeline.convert_image
<scrapy.pipelines.images.ImagesPipeline.convert_image>` must now accept a
``response_body`` parameter.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
New features
~~~~~~~~~~~~
- Applied black_ coding style to files generated with the
:command:`genspider` and :command:`startproject` commands.
(:issue:`5809`, :issue:`5814`)
.. _black: https://black.readthedocs.io/en/stable/
- :setting:`FEED_EXPORT_ENCODING` is now set to ``"utf-8"`` in the
``settings.py`` file that the :command:`startproject` command generates.
With this value, JSON exports won’t force the use of escape sequences for
non-ASCII characters.
(:issue:`5797`, :issue:`5800`)
- The :class:`~scrapy.extensions.memusage.MemoryUsage` extension now logs the
peak memory usage during checks, and the binary unit MiB is now used to
avoid confusion.
(:issue:`5717`, :issue:`5722`, :issue:`5727`)
- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set
to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from
``None``, as the latter indicates that the default spider callback
(:meth:`~scrapy.Spider.parse`) is to be used.
(:issue:`5798`)
Bug fixes
~~~~~~~~~
- Enabled unsafe legacy SSL renegotiation to fix access to some outdated
websites.
(:issue:`5491`, :issue:`5790`)
- Fixed STARTTLS-based email delivery not working with Twisted 21.2.0 and
better.
(:issue:`5386`, :issue:`5406`)
- Fixed the :meth:`finish_exporting` method of :ref:`item exporters
<topics-exporters>` not being called for empty files.
(:issue:`5537`, :issue:`5758`)
- Fixed HTTP/2 responses getting only the last value for a header when
multiple headers with the same name are received.
(:issue:`5777`)
- Fixed an exception raised by the :command:`shell` command on some cases
when :ref:`using asyncio <using-asyncio>`.
(:issue:`5740`, :issue:`5742`, :issue:`5748`, :issue:`5759`, :issue:`5760`,
:issue:`5771`)
- When using :class:`~scrapy.spiders.CrawlSpider`, callback keyword arguments
(``cb_kwargs``) added to a request in the ``process_request`` callback of a
:class:`~scrapy.spiders.Rule` will no longer be ignored.
(:issue:`5699`)
- The :ref:`images pipeline <images-pipeline>` no longer re-encodes JPEG
files.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
- Fixed the handling of transparent WebP images by the :ref:`images pipeline
<images-pipeline>`.
(:issue:`3072`, :issue:`5766`, :issue:`5767`)
- :func:`scrapy.shell.inspect_response` no longer inhibits ``SIGINT``
(Ctrl+C).
(:issue:`2918`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
with ``unique=False`` no longer filters out links that have identical URL
*and* text.
(:issue:`3798`, :issue:`3799`, :issue:`4695`, :issue:`5458`)
- :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` now
ignores URL protocols that do not support ``robots.txt`` (``data://``,
``file://``).
(:issue:`5807`)
- Silenced the ``filelock`` debug log messages introduced in Scrapy 2.6.
(:issue:`5753`, :issue:`5754`)
- Fixed the output of ``scrapy -h`` showing an unintended ``**commands**``
line.
(:issue:`5709`, :issue:`5711`, :issue:`5712`)
- Made the active project indication in the output of :ref:`commands
<topics-commands>` more clear.
(:issue:`5715`)
Documentation
~~~~~~~~~~~~~
- Documented how to :ref:`debug spiders from Visual Studio Code
<debug-vscode>`.
(:issue:`5721`)
- Documented how :setting:`DOWNLOAD_DELAY` affects per-domain concurrency.
(:issue:`5083`, :issue:`5540`)
- Improved consistency.
(:issue:`5761`)
- Fixed typos.
(:issue:`5714`, :issue:`5744`, :issue:`5764`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Applied :ref:`black coding style <coding-style>`, sorted import statements,
and introduced :ref:`pre-commit <scrapy-pre-commit>`.
(:issue:`4654`, :issue:`4658`, :issue:`5734`, :issue:`5737`, :issue:`5806`,
:issue:`5810`)
- Switched from :mod:`os.path` to :mod:`pathlib`.
(:issue:`4916`, :issue:`4497`, :issue:`5682`)
- Addressed many issues reported by Pylint.
(:issue:`5677`)
- Improved code readability.
(:issue:`5736`)
- Improved package metadata.
(:issue:`5768`)
- Removed direct invocations of ``setup.py``.
(:issue:`5774`, :issue:`5776`)
- Removed unnecessary :class:`~collections.OrderedDict` usages.
(:issue:`5795`)
- Removed unnecessary ``__str__`` definitions.
(:issue:`5150`)
- Removed obsolete code and comments.
(:issue:`5725`, :issue:`5729`, :issue:`5730`, :issue:`5732`)
- Fixed test and CI issues.
(:issue:`5749`, :issue:`5750`, :issue:`5756`, :issue:`5762`, :issue:`5765`,
:issue:`5780`, :issue:`5781`, :issue:`5782`, :issue:`5783`, :issue:`5785`,
:issue:`5786`)
.. _release-2.7.1:
Scrapy 2.7.1 (2022-11-02)
@ -2417,11 +3050,13 @@ Backward-incompatible changes
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
into lists:
>>> item = MyItem()
>>> item['field'] = 'value1'
>>> loader = ItemLoader(item=item)
>>> item['field']
['value1']
.. code-block:: pycon
>>> item = MyItem()
>>> item["field"] = "value1"
>>> loader = ItemLoader(item=item)
>>> item["field"]
['value1']
This is needed to allow adding values to existing fields
(``loader.add_value('field', 'value2')``).
@ -3999,8 +4634,6 @@ Relocations
+ Note: telnet is not enabled on Python 3
(https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
.. _parsel: https://github.com/scrapy/parsel
Bugfixes
~~~~~~~~
@ -4700,7 +5333,7 @@ Scrapy 0.22.1 (released 2014-02-08)
- BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
- BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
- Fix tests for Travis-CI build (:commit:`76c7e20`)
- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
- Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
- Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
@ -4725,7 +5358,7 @@ Enhancements
- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
- Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`)
- Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
- Selectors register EXSLT namespaces by default (:issue:`472`)
- Unify item loaders similar to selectors renaming (:issue:`461`)
@ -4905,7 +5538,7 @@ Scrapy 0.18.0 (released 2013-08-09)
-----------------------------------
- Lot of improvements to testsuite run using Tox, including a way to test on pypi
- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`)
- Use lxml recover option to parse sitemaps (:issue:`347`)
- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
@ -4939,8 +5572,8 @@ Scrapy 0.18.0 (released 2013-08-09)
- Added ``--pdb`` option to ``scrapy`` command line tool
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
- Several improvements to spider contracts
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections,
- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62
- added from_crawler method to spiders
- added system tests with mock server
- more improvements to macOS compatibility (thanks Alex Cepoi)
@ -5082,7 +5715,7 @@ Scrapy changes:
- promoted :ref:`topics-djangoitem` to main contrib
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- removed signal: ``scrapy.mail.mail_sent``
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
- DBM is now the default storage backend for HTTP cache middleware
@ -5148,7 +5781,7 @@ Scrapy 0.14
New features and settings
~~~~~~~~~~~~~~~~~~~~~~~~~
- Support for `AJAX crawleable urls`_
- Support for `AJAX crawlable urls`_
- New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
- added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
- Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
@ -5408,7 +6041,7 @@ Backward-incompatible changes
- Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
- Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
- Refactored HTTP Cache middleware
- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
- Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
- Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
- Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
@ -5419,7 +6052,8 @@ Scrapy 0.7
First release of Scrapy.
.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _boto3: https://github.com/boto/boto3
.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
@ -5430,6 +6064,7 @@ First release of Scrapy.
.. _LevelDB: https://github.com/google/leveldb
.. _lxml: https://lxml.de/
.. _marshal: https://docs.python.org/2/library/marshal.html
.. _parsel: https://github.com/scrapy/parsel
.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr

193
docs/topics/addons.rst Normal file
View File

@ -0,0 +1,193 @@
.. _topics-addons:
=======
Add-ons
=======
Scrapy's add-on system is a framework which unifies managing and configuring
components that extend Scrapy's core functionality, such as middlewares,
extensions, or pipelines. It provides users with a plug-and-play experience in
Scrapy extension management, and grants extensive configuration control to
developers.
Activating and configuring add-ons
==================================
During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled
add-ons is read from your ``ADDONS`` setting.
The ``ADDONS`` setting is a dict in which every key is an add-on class or its
import path and the value is its priority.
This is an example where two add-ons are enabled in a project's
``settings.py``::
ADDONS = {
'path.to.someaddon': 0,
SomeAddonClass: 1,
}
Writing your own add-ons
========================
Add-ons are Python classes that include the following method:
.. method:: update_settings(settings)
This method is called during the initialization of the
:class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks
(e.g. for external Python libraries) and update the
:class:`~scrapy.settings.Settings` object as wished, e.g. enable components
for this add-on or set required configuration of other extensions.
:param settings: The settings object storing Scrapy/component configuration
:type settings: :class:`~scrapy.settings.Settings`
They can also have the following method:
.. classmethod:: from_crawler(cls, crawler)
:noindex:
If present, this class method is called to create an add-on instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the add-on. The crawler object provides access to all Scrapy core
components like settings and signals; it is a way for the add-on to access
them and hook its functionality into Scrapy.
:param crawler: The crawler that uses this add-on
:type crawler: :class:`~scrapy.crawler.Crawler`
The settings set by the add-on should use the ``addon`` priority (see
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
class MyAddon:
def update_settings(self, settings):
settings.set("DNSCACHE_ENABLED", True, "addon")
This allows users to override these settings in the project or spider
configuration. This is not possible with settings that are mutable objects,
such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases
you can provide an add-on-specific setting that governs whether the add-on will
modify :setting:`ITEM_PIPELINES`::
class MyAddon:
def update_settings(self, settings):
if settings.getbool("MYADDON_ENABLE_PIPELINE"):
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
If the ``update_settings`` method raises
:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes
it easy to enable an add-on only when some conditions are met.
Fallbacks
---------
Some components provided by add-ons need to fall back to "default"
implementations, e.g. a custom download handler needs to send the request that
it doesn't handle via the default download handler, or a stats collector that
includes some additional processing but otherwise uses the default stats
collector. And it's possible that a project needs to use several custom
components of the same type, e.g. two custom download handlers that support
different kinds of custom requests and still need to use the default download
handler for other requests. To make such use cases easier to configure, we
recommend that such custom components should be written in the following way:
1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the
default Scrapy one (e.g.
``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead
be able to load the class of the fallback component from a special setting
(e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use
it.
2. The add-ons that include these components should read the current value of
the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their
``update_settings()`` methods, save that value into the fallback setting
(``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default
setting to the component provided by the add-on (e.g.
``MyDownloadHandler``). If the fallback setting is already set by the user,
they shouldn't change it.
3. This way, if there are several add-ons that want to modify the same setting,
all of them will fallback to the component from the previous one and then to
the Scrapy default. The order of that depends on the priority order in the
``ADDONS`` setting.
Add-on examples
===============
Set some basic configuration:
.. code-block:: python
class MyAddon:
def update_settings(self, settings):
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
settings.set("DNSCACHE_ENABLED", True, "addon")
Check dependencies:
.. code-block:: python
class MyAddon:
def update_settings(self, settings):
try:
import boto
except ImportError:
raise NotConfigured("MyAddon requires the boto library")
...
Access the crawler instance:
.. code-block:: python
class MyAddon:
def __init__(self, crawler) -> None:
super().__init__()
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def update_settings(self, settings):
...
Use a fallback component:
.. code-block:: python
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
class MyHandler:
lazy = False
def __init__(self, settings, crawler):
dhcls = load_object(settings.get(FALLBACK_SETTING))
self._fallback_handler = create_instance(
dhcls,
settings=None,
crawler=crawler,
)
def download_request(self, request, spider):
if request.meta.get("my_params"):
# handle the request
...
else:
return self._fallback_handler.download_request(request, spider)
class MyAddon:
def update_settings(self, settings):
if not settings.get(FALLBACK_SETTING):
settings.set(
FALLBACK_SETTING,
settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
"addon",
)
settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler

17
docs/topics/api.rst
View File

@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares
Starts the crawler by instantiating its spider class with the given
``args`` and ``kwargs`` arguments, while setting the execution engine in
motion.
motion. Should be called only once.
Returns a deferred that is fired when the crawl is finished.
@ -132,16 +132,15 @@ Settings API
precedence over lesser ones when setting and retrieving values in the
:class:`~scrapy.settings.Settings` class.
.. highlight:: python
::
.. code-block:: python
SETTINGS_PRIORITIES = {
'default': 0,
'command': 10,
'project': 20,
'spider': 30,
'cmdline': 40,
"default": 0,
"command": 10,
"addon": 15,
"project": 20,
"spider": 30,
"cmdline": 40,
}
For a detailed explanation on each settings sources, see:

106
docs/topics/asyncio.rst
View File

@ -27,54 +27,43 @@ reactor manually. You can do that using
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
.. _asyncio-preinstalled-reactor:
Using custom asyncio loops
==========================
Handling a pre-installed reactor
================================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
use it instead of the default asyncio event loop.
``twisted.internet.reactor`` and some other Twisted imports install the default
Twisted reactor as a side effect. Once a Twisted reactor is installed, it is
not possible to switch to a different reactor at run time.
If you :ref:`configure the asyncio Twisted reactor <install-asyncio>` and, at
run time, Scrapy complains that a different reactor is already installed,
chances are you have some such imports in your code.
You can usually fix the issue by moving those offending module-level Twisted
imports to the method or function definitions where they are used. For example,
if you have something like:
.. code-block:: python
from twisted.internet import reactor
.. _asyncio-windows:
def my_function():
reactor.callLater(...)
Windows-specific notes
======================
Switch to something like:
The Windows implementation of :mod:`asyncio` can use two event loop
implementations:
.. code-block:: python
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
when using Twisted.
def my_function():
from twisted.internet import reactor
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
with Twisted.
reactor.callLater(...)
So on Python 3.8+ the event loop class needs to be changed.
.. versionchanged:: 2.6.0
The event loop class is changed automatically when you change the
:setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`.
To change the event loop class manually, call the following code before
installing the reactor::
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
You can put this in the same function that installs the reactor, if you do that
yourself, or in some code that runs before the reactor is installed, e.g.
``settings.py``.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
Alternatively, you can try to :ref:`manually install the asyncio reactor
<install-asyncio>`, with :func:`~scrapy.utils.reactor.install_reactor`, before
those imports happen.
.. _asyncio-await-dfd:
@ -106,12 +95,14 @@ Enforcing asyncio as a requirement
If you are writing a :ref:`component <topics-components>` that requires asyncio
to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
:ref:`enforce it as a requirement <enforce-component-requirements>`. For
example::
example:
.. code-block:: python
from scrapy.utils.reactor import is_asyncio_reactor_installed
class MyComponent:
class MyComponent:
def __init__(self):
if not is_asyncio_reactor_installed():
raise ValueError(
@ -120,3 +111,36 @@ example::
f"TWISTED_REACTOR setting. See the asyncio documentation "
f"of Scrapy for more information."
)
.. _asyncio-windows:
Windows-specific notes
======================
The Windows implementation of :mod:`asyncio` can use two event loop
implementations, :class:`~asyncio.ProactorEventLoop` (default) and
:class:`~asyncio.SelectorEventLoop`. However, only
:class:`~asyncio.SelectorEventLoop` works with Twisted.
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
.. _using-custom-loops:
Using custom asyncio loops
==========================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event
loop class to use it instead of the default asyncio event loop.

40
docs/topics/broad-crawls.rst
View File

@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ
It works best during single-domain crawl. It does not work well with crawling
many different domains in parallel
To apply the recommended priority queue use::
To apply the recommended priority queue use:
SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
.. code-block:: python
SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"
.. _broad-crawls-concurrency:
@ -71,7 +73,9 @@ many different domains in parallel, so you will want to increase it. How much
to increase it will depend on how much CPU and memory your crawler will have
available.
A good starting point is ``100``::
A good starting point is ``100``:
.. code-block:: python
CONCURRENT_REQUESTS = 100
@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of
threads handling DNS queries. The DNS queue will be processed faster speeding
up establishing of connection and crawling overall.
To increase maximum thread pool size use::
To increase maximum thread pool size use:
.. code-block:: python
REACTOR_THREADPOOL_MAXSIZE = 20
@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in
production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::
To set the log level use:
LOG_LEVEL = 'INFO'
.. code-block:: python
LOG_LEVEL = "INFO"
Disable cookies
===============
@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve
performance by saving some CPU cycles and reducing the memory footprint of your
Scrapy crawler.
To disable cookies use::
To disable cookies use:
.. code-block:: python
COOKIES_ENABLED = False
@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout
error which gets retried many times, unnecessarily, preventing crawler capacity
to be reused for other domains.
To disable retries use::
To disable retries use:
.. code-block:: python
RETRY_ENABLED = False
@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the
case for broad crawls) reduce the download timeout so that stuck requests are
discarded quickly and free up capacity to process the next ones.
To reduce the download timeout use::
To reduce the download timeout use:
.. code-block:: python
DOWNLOAD_TIMEOUT = 15
@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of
request constant per crawl batch, otherwise redirect loops may cause the
crawler to dedicate too many resources on any specific domain.
To disable redirects use::
To disable redirects use:
.. code-block:: python
REDIRECT_ENABLED = False
@ -179,7 +195,9 @@ Pages can indicate it in two ways:
"main", "index" website pages.
Scrapy handles (1) automatically; to handle (2) enable
:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`::
:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`:
.. code-block:: python
AJAXCRAWL_ENABLED = True

26
docs/topics/commands.rst
View File

@ -238,9 +238,6 @@ genspider
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. note:: Even if an HTTPS URL is specified, the protocol used in
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
Usage example::
$ scrapy genspider -l
@ -288,13 +285,13 @@ Usage examples::
$ scrapy crawl myspider
[ ... myspider starts crawling ... ]
$ scrapy -o myfile:csv myspider
$ scrapy crawl -o myfile:csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
$ scrapy -O myfile:json myspider
$ scrapy crawl -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
$ scrapy -o myfile -t csv myspider
$ scrapy crawl -o myfile -t csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check
@ -617,7 +614,7 @@ Example:
.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
COMMANDS_MODULE = "mybot.commands"
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
@ -636,10 +633,11 @@ The following example adds ``my_command`` command:
from setuptools import setup, find_packages
setup(name='scrapy-mymodule',
entry_points={
'scrapy.commands': [
'my_command=my_scrapy_module.commands:MyCommand',
],
},
)
setup(
name="scrapy-mymodule",
entry_points={
"scrapy.commands": [
"my_command=my_scrapy_module.commands:MyCommand",
],
},
)

10
docs/topics/components.rst
View File

@ -66,16 +66,18 @@ version mismatch, while :exc:`ValueError` may be better if the issue is the
value of a setting.
If your requirement is a minimum Scrapy version, you may use
:attr:`scrapy.__version__` to enforce your requirement. For example::
:attr:`scrapy.__version__` to enforce your requirement. For example:
from pkg_resources import parse_version
.. code-block:: python
from packaging.version import parse as parse_version
import scrapy
class MyComponent:
class MyComponent:
def __init__(self):
if parse_version(scrapy.__version__) < parse_version('2.7'):
if parse_version(scrapy.__version__) < parse_version("2.7"):
raise RuntimeError(
f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
f"later, which allow defining the process_spider_output "

39
docs/topics/contracts.rst
View File

@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts.
This allows you to test each callback of your spider by hardcoding a sample url
and check various constraints for how the callback processes the response. Each
contract is prefixed with an ``@`` and included in the docstring. See the
following example::
following example:
.. code-block:: python
def parse(self, response):
""" This function parses a sample response. Some contracts are mingled
"""
This function parses a sample response. Some contracts are mingled
with this docstring.
@url http://www.amazon.com/s?field-keywords=selfish+gene
@ -64,11 +67,13 @@ Custom Contracts
If you find you need more power than the built-in Scrapy contracts you can
create and load your own contracts in the project by using the
:setting:`SPIDER_CONTRACTS` setting::
:setting:`SPIDER_CONTRACTS` setting:
.. code-block:: python
SPIDER_CONTRACTS = {
'myproject.contracts.ResponseCheck': 10,
'myproject.contracts.ItemValidate': 10,
"myproject.contracts.ResponseCheck": 10,
"myproject.contracts.ItemValidate": 10,
}
Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
@ -111,22 +116,27 @@ Raise :class:`~scrapy.exceptions.ContractFail` from
.. autoclass:: scrapy.exceptions.ContractFail
Here is a demo contract which checks the presence of a custom header in the
response received::
response received:
.. skip: next
.. code-block:: python
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
class HasHeaderContract(Contract):
""" Demo contract which checks the presence of a custom header
@has_header X-CustomHeader
"""
Demo contract which checks the presence of a custom header
@has_header X-CustomHeader
"""
name = 'has_header'
name = "has_header"
def pre_process(self, response):
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
raise ContractFail("X-CustomHeader not present")
.. _detecting-contract-check-runs:
@ -135,14 +145,17 @@ Detecting check runs
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
your spiders or your settings when ``scrapy check`` is used::
your spiders or your settings when ``scrapy check`` is used:
.. code-block:: python
import os
import scrapy
class ExampleSpider(scrapy.Spider):
name = 'example'
name = "example"
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
if os.environ.get("SCRAPY_CHECK"):
pass # Do some scraper adjustments when a check is running

87
docs/topics/coroutines.rst
View File

@ -58,49 +58,59 @@ There are several use cases for coroutines in Scrapy.
Code that would return Deferreds when written for previous Scrapy versions,
such as downloader middlewares and signal handlers, can be rewritten to be
shorter and cleaner::
shorter and cleaner:
.. code-block:: python
from itemadapter import ItemAdapter
class DbPipeline:
def _update_item(self, data, item):
adapter = ItemAdapter(item)
adapter['field'] = data
adapter["field"] = data
return item
def process_item(self, item, spider):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter['id'])
dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
becomes:
.. code-block:: python
from itemadapter import ItemAdapter
class DbPipeline:
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
adapter['field'] = await db.get_some_data(adapter['id'])
adapter["field"] = await db.get_some_data(adapter["id"])
return item
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
This means you can use many useful Python libraries providing such code:
.. skip: next
.. code-block:: python
class MySpiderDeferred(Spider):
# ...
async def parse(self, response):
additional_response = await treq.get('https://additional.url')
additional_response = await treq.get("https://additional.url")
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
class MySpiderAsyncio(Spider):
# ...
async def parse(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
async with session.get("https://additional.url") as additional_response:
additional_data = await additional_response.text()
# ... use response and additional_data to yield items and requests
@ -124,6 +134,63 @@ Common use cases for asynchronous code include:
.. _aio-libs: https://github.com/aio-libs
.. _inline-requests:
Inline requests
===============
The spider below shows how to send a request and await its response all from
within a spider callback:
.. code-block:: python
from scrapy import Spider, Request
from scrapy.utils.defer import maybe_deferred_to_future
class SingleRequestSpider(Spider):
name = "single"
start_urls = ["https://example.org/product"]
async def parse(self, response, **kwargs):
additional_request = Request("https://example.org/price")
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
yield {
"h1": response.css("h1").get(),
"price": additional_response.css("#price").get(),
}
You can also send multiple requests in parallel:
.. code-block:: python
from scrapy import Spider, Request
from scrapy.utils.defer import maybe_deferred_to_future
from twisted.internet.defer import DeferredList
class MultipleRequestsSpider(Spider):
name = "multiple"
start_urls = ["https://example.com/product"]
async def parse(self, response, **kwargs):
additional_requests = [
Request("https://example.com/price"),
Request("https://example.com/color"),
]
deferreds = []
for r in additional_requests:
deferred = self.crawler.engine.download(r)
deferreds.append(deferred)
responses = await maybe_deferred_to_future(DeferredList(deferreds))
yield {
"h1": response.css("h1::text").get(),
"price": responses[0][1].css(".price::text").get(),
"price2": responses[1][1].css(".color::text").get(),
}
.. _sync-async-spider-middleware:
Mixing synchronous and asynchronous spider middlewares
@ -192,7 +259,9 @@ while maintaining support for older Scrapy versions, you may define
:term:`asynchronous generator` version of that method with an alternative name:
``process_spider_output_async``.
For example::
For example:
.. code-block:: python
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider):

38
docs/topics/debug.rst
View File

@ -5,21 +5,25 @@ Debugging Spiders
=================
This document explains the most common techniques for debugging spiders.
Consider the following Scrapy spider below::
Consider the following Scrapy spider below:
.. skip: next
.. code-block:: python
import scrapy
from myproject.items import MyItem
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
start_urls = (
'http://example.com/page1',
'http://example.com/page2',
)
"http://example.com/page1",
"http://example.com/page2",
)
def parse(self, response):
# <processing code not shown>
# collect `item_urls`
# collect `item_urls`
for item_url in item_urls:
yield scrapy.Request(item_url, self.parse_item)
@ -28,7 +32,9 @@ Consider the following Scrapy spider below::
item = MyItem()
# populate `item` fields
# and extract item_details_url
yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
yield scrapy.Request(
item_details_url, self.parse_details, cb_kwargs={"item": item}
)
def parse_details(self, response, item):
# populate more `item` fields
@ -103,10 +109,13 @@ showing the response received and the output. How to debug the situation when
.. highlight:: python
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
:ref:`topics-shell-inspect-response`):
.. code-block:: python
from scrapy.shell import inspect_response
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
@ -121,10 +130,13 @@ Open in browser
Sometimes you just want to see how a certain response looks in a browser, you
can use the ``open_in_browser`` function for that. Here is an example of how
you would use it::
you would use it:
.. code-block:: python
from scrapy.utils.response import open_in_browser
def parse_details(self, response):
if "item name" not in response.body:
open_in_browser(response)
@ -138,19 +150,23 @@ Logging
Logging is another useful option for getting information about your spider run.
Although not as convenient, it comes with the advantage that the logs will be
available in all future runs should they be necessary again::
available in all future runs should they be necessary again:
.. code-block:: python
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
else:
self.logger.warning('No item received for %s', response.url)
self.logger.warning("No item received for %s", response.url)
For more information, check the :ref:`topics-logging` section.
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
.. _debug-vscode:
Visual Studio Code
==================

35
docs/topics/developer-tools.rst
View File

@ -94,8 +94,10 @@ Then, back to your web browser, right-click on the ``span`` tag, select
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
.. code-block:: pycon
>>> response.xpath("/html/body/div/div[2]/div[1]/div[1]/span[1]/text()").getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@ -124,11 +126,13 @@ With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
.. code-block:: pycon
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@ -237,17 +241,19 @@ on the request and open ``Open in new tab`` to get a better overview.
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
also request each page to get every quote on the site:
.. code-block:: python
import scrapy
import json
class QuoteSpider(scrapy.Spider):
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
name = "quote"
allowed_domains = ["quotes.toscrape.com"]
page = 1
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
def parse(self, response):
data = json.loads(response.text)
@ -275,7 +281,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
format, by right-clicking on each of them in the network tool and using the
:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
request::
request:
.. code-block:: python
from scrapy import Request
@ -286,7 +294,8 @@ request::
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
"zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
"I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'"
)
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`

87
docs/topics/downloader-middleware.rst
View File

@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the
:setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the
middleware class paths and their values are the middleware orders.
Here's an example::
Here's an example:
.. code-block:: python
DOWNLOADER_MIDDLEWARES = {
'myproject.middlewares.CustomDownloaderMiddleware': 543,
"myproject.middlewares.CustomDownloaderMiddleware": 543,
}
The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied.
If you want to disable a built-in middleware (the ones defined in
:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None``
as its value. For example, if you want to disable the user-agent middleware::
as its value. For example, if you want to disable the user-agent middleware:
.. code-block:: python
DOWNLOADER_MIDDLEWARES = {
'myproject.middlewares.CustomDownloaderMiddleware': 543,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
"myproject.middlewares.CustomDownloaderMiddleware": 543,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -226,20 +230,26 @@ There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
For example::
For example:
.. skip: next
.. code-block:: python
for i, url in enumerate(urls):
yield scrapy.Request(url, meta={'cookiejar': i},
callback=self.parse_page)
yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page)
Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep
passing it along on subsequent requests. For example::
passing it along on subsequent requests. For example:
.. code-block:: python
def parse_page(self, response):
# do some processing
return scrapy.Request("http://www.example.com/otherpage",
meta={'cookiejar': response.meta['cookiejar']},
callback=self.parse_other_page)
return scrapy.Request(
"http://www.example.com/otherpage",
meta={"cookiejar": response.meta["cookiejar"]},
callback=self.parse_other_page,
)
.. setting:: COOKIES_ENABLED
@ -339,16 +349,18 @@ HttpAuthMiddleware
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example::
Example:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class SomeIntranetSiteSpider(CrawlSpider):
http_user = 'someuser'
http_pass = 'somepass'
http_auth_domain = 'intranet.example.com'
name = 'intranet.example.com'
class SomeIntranetSiteSpider(CrawlSpider):
http_user = "someuser"
http_pass = "somepass"
http_auth_domain = "intranet.example.com"
name = "intranet.example.com"
# .. rest of the spider code omitted ...
@ -792,7 +804,9 @@ If you want to handle some redirect status codes in your spider, you can
specify these in the ``handle_httpstatus_list`` spider attribute.
For example, if you want the redirect middleware to ignore 301 and 302
responses (and pass them through to your spider) you can do this::
responses (and pass them through to your spider) you can do this:
.. code-block:: python
class MySpider(CrawlSpider):
handle_httpstatus_list = [301, 302]
@ -901,6 +915,7 @@ settings (see the settings documentation for more info):
* :setting:`RETRY_ENABLED`
* :setting:`RETRY_TIMES`
* :setting:`RETRY_HTTP_CODES`
* :setting:`RETRY_EXCEPTIONS`
.. reqmeta:: dont_retry
@ -952,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_EXCEPTIONS
RETRY_EXCEPTIONS
^^^^^^^^^^^^^^^^
Default::
[
'twisted.internet.defer.TimeoutError',
'twisted.internet.error.TimeoutError',
'twisted.internet.error.DNSLookupError',
'twisted.internet.error.ConnectionRefusedError',
'twisted.internet.error.ConnectionDone',
'twisted.internet.error.ConnectError',
'twisted.internet.error.ConnectionLost',
'twisted.internet.error.TCPTimedOutError',
'twisted.web.client.ResponseFailed',
IOError,
'scrapy.core.downloader.handlers.http11.TunnelError',
]
List of exceptions to retry.
Each list entry may be an exception type or its import path as a string.
An exception will not be caught when the exception type is not in
:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
exception propagation, see
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
@ -993,8 +1039,8 @@ RobotsTxtMiddleware
* :ref:`Protego <protego-parser>` (default)
* :ref:`RobotFileParser <python-robotfileparser>`
* :ref:`Reppy <reppy-parser>`
* :ref:`Robotexclusionrulesparser <rerp-parser>`
* :ref:`Reppy <reppy-parser>` (deprecated)
You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@ -1087,6 +1133,7 @@ In order to use this parser:
.. warning:: `Upstream issue #122
<https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
Because of this the Reppy parser is deprecated.
* Set :setting:`ROBOTSTXT_PARSER` setting to
``scrapy.robotstxt.ReppyRobotParser``

57
docs/topics/dynamic-content.rst
View File

@ -119,16 +119,20 @@ data from it depends on the type of response:
<topics-selectors>` as usual.
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
:attr:`response.text <scrapy.http.TextResponse.text>`:
.. code-block:: python
data = json.loads(response.text)
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a
:class:`~scrapy.Selector` and then
:ref:`use it <topics-selectors>` as usual::
:ref:`use it <topics-selectors>` as usual:
selector = Selector(data['html'])
.. code-block:: python
selector = Selector(data["html"])
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
@ -179,10 +183,12 @@ data from it:
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
>>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
>>> json_data = response.css('script::text').re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
.. code-block:: pycon
>>> pattern = r"\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n"
>>> json_data = response.css("script::text").re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
@ -190,11 +196,13 @@ data from it:
``var data = {field: "value", secondField: "second value"};``
you can extract that data as follows:
>>> import chompjs
>>> javascript = response.css('script::text').get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
.. code-block:: pycon
>>> import chompjs
>>> javascript = response.css("script::text").get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
@ -202,14 +210,16 @@ data from it:
For example, if the JavaScript code contains
``var data = {field: "value"};`` you can extract that data as follows:
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css('script::text').get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. code-block:: pycon
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css("script::text").get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding="unicode")
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. _topics-javascript-rendering:
@ -250,11 +260,14 @@ automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
One such library is `playwright-python`_ (an official Python port of `playwright`_).
The following is a simple snippet to illustrate its usage within a Scrapy spider::
The following is a simple snippet to illustrate its usage within a Scrapy spider:
.. code-block:: python
import scrapy
from playwright.async_api import async_playwright
class PlaywrightSpider(scrapy.Spider):
name = "playwright"
start_urls = ["data:,"] # avoid using the default Scrapy downloader
@ -263,7 +276,7 @@ The following is a simple snippet to illustrate its usage within a Scrapy spider
async with async_playwright() as pw:
browser = await pw.chromium.launch()
page = await browser.new_page()
await page.goto("https:/example.org")
await page.goto("https://example.org")
title = await page.title()
return {"title": title}

22
docs/topics/email.rst
View File

@ -19,19 +19,33 @@ Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
the standard ``__init__`` method::
the standard ``__init__`` method:
.. code-block:: python
from scrapy.mail import MailSender
mailer = MailSender()
Or you can instantiate it passing a Scrapy settings object, which will respect
the :ref:`settings <topics-email-settings>`::
the :ref:`settings <topics-email-settings>`:
.. skip: start
.. code-block:: python
mailer = MailSender.from_settings(settings)
And here is how to use it to send an e-mail (without attachments)::
And here is how to use it to send an e-mail (without attachments):
mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
.. code-block:: python
mailer.send(
to=["someone@example.com"],
subject="Some subject",
body="Some body",
cc=["another@example.com"],
)
.. skip: end
MailSender class reference
==========================

8
docs/topics/exceptions.rst
View File

@ -26,11 +26,13 @@ CloseSpider
:param reason: the reason for closing
:type reason: str
For example::
For example:
.. code-block:: python
def parse_page(self, response):
if 'Bandwidth exceeded' in response.body:
raise CloseSpider('bandwidth_exceeded')
if "Bandwidth exceeded" in response.body:
raise CloseSpider("bandwidth_exceeded")
DontCloseSpider
---------------

36
docs/topics/exporters.rst
View File

@ -38,11 +38,14 @@ the end of the exporting process
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
value of one of their fields:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
@ -56,9 +59,9 @@ value of one of their fields::
def _exporter_for_item(self, item):
adapter = ItemAdapter(item)
year = adapter['year']
year = adapter["year"]
if year not in self.year_to_exporter:
xml_file = open(f'{year}.xml', 'wb')
xml_file = open(f"{year}.xml", "wb")
exporter = XmlItemExporter(xml_file)
exporter.start_exporting()
self.year_to_exporter[year] = (exporter, xml_file)
@ -94,12 +97,16 @@ If you use :class:`~scrapy.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
Example:
.. code-block:: python
import scrapy
def serialize_price(value):
return f'$ {str(value)}'
return f"$ {str(value)}"
class Product(scrapy.Item):
name = scrapy.Field()
@ -115,15 +122,17 @@ customize how your field value will be exported.
Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method
after your custom code.
Example::
Example:
.. code-block:: python
from scrapy.exporters import XmlItemExporter
class ProductXmlExporter(XmlItemExporter):
class ProductXmlExporter(XmlItemExporter):
def serialize_field(self, field, name, value):
if name == 'price':
return f'$ {str(value)}'
if name == "price":
return f"$ {str(value)}"
return super().serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -132,10 +141,13 @@ Built-in Item Exporters reference
=================================
Here is a list of the Item Exporters bundled with Scrapy. Some of them contain
output examples, which assume you're exporting these two items::
output examples, which assume you're exporting these two items:
Item(name='Color TV', price='1200')
Item(name='DVD player', price='200')
.. skip: next
.. code-block:: python
Item(name="Color TV", price="1200")
Item(name="DVD player", price="200")
BaseItemExporter
----------------

153
docs/topics/extensions.rst
View File

@ -31,11 +31,13 @@ initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
by a string: the full Python path to the extension's class name. For example::
by a string: the full Python path to the extension's class name. For example:
.. code-block:: python
EXTENSIONS = {
'scrapy.extensions.corestats.CoreStats': 500,
'scrapy.extensions.telnet.TelnetConsole': 500,
"scrapy.extensions.corestats.CoreStats": 500,
"scrapy.extensions.telnet.TelnetConsole": 500,
}
@ -64,10 +66,12 @@ Disabling an extension
In order to disable an extension that comes enabled by default (i.e. those
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
``None``. For example::
``None``. For example:
.. code-block:: python
EXTENSIONS = {
'scrapy.extensions.corestats.CoreStats': None,
"scrapy.extensions.corestats.CoreStats": None,
}
Writing your own extension
@ -98,7 +102,9 @@ in the previous section. This extension will log a message every time:
The extension will be enabled through the ``MYEXT_ENABLED`` setting and the
number of items will be specified through the ``MYEXT_ITEMCOUNT`` setting.
Here is the code of such extension::
Here is the code of such extension:
.. code-block:: python
import logging
from scrapy import signals
@ -106,8 +112,8 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging:
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count
self.items_scraped = 0
@ -116,11 +122,11 @@ Here is the code of such extension::
def from_crawler(cls, crawler):
# first check if the extension should be enabled and raise
# NotConfigured otherwise
if not crawler.settings.getbool('MYEXT_ENABLED'):
if not crawler.settings.getbool("MYEXT_ENABLED"):
raise NotConfigured
# get the number of items from settings
item_count = crawler.settings.getint('MYEXT_ITEMCOUNT', 1000)
item_count = crawler.settings.getint("MYEXT_ITEMCOUNT", 1000)
# instantiate the extension object
ext = cls(item_count)
@ -252,6 +258,7 @@ The conditions for closing a spider can be configured through the following
settings:
* :setting:`CLOSESPIDER_TIMEOUT`
* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
* :setting:`CLOSESPIDER_ITEMCOUNT`
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
@ -274,6 +281,18 @@ more than that number of second, it will be automatically closed with the
reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
timeout.
.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
CLOSESPIDER_TIMEOUT_NO_ITEM
"""""""""""""""""""""""""""
Default: ``0``
An integer which specifies a number of seconds. If the spider has not produced
any items in the last number of seconds, it will be closed with the reason
``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed
regardless if it hasn't produced any items.
.. setting:: CLOSESPIDER_ITEMCOUNT
CLOSESPIDER_ITEMCOUNT
@ -331,6 +350,122 @@ full list of parameters, including examples on how to instantiate
.. module:: scrapy.extensions.debug
:synopsis: Extensions for debugging Scrapy
.. module:: scrapy.extensions.periodic_log
:synopsis: Periodic stats logging
Periodic log extension
~~~~~~~~~~~~~~~~~~~~~~
.. class:: PeriodicLog
This extension periodically logs rich stat data as a JSON object::
2023-08-04 02:30:57 [scrapy.extensions.logstats] INFO: Crawled 976 pages (at 162 pages/min), scraped 925 items (at 161 items/min)
2023-08-04 02:30:57 [scrapy.extensions.periodic_log] INFO: {
"delta": {
"downloader/request_bytes": 55582,
"downloader/request_count": 162,
"downloader/request_method_count/GET": 162,
"downloader/response_bytes": 618133,
"downloader/response_count": 162,
"downloader/response_status_count/200": 162,
"item_scraped_count": 161
},
"stats": {
"downloader/request_bytes": 338243,
"downloader/request_count": 992,
"downloader/request_method_count/GET": 992,
"downloader/response_bytes": 3836736,
"downloader/response_count": 976,
"downloader/response_status_count/200": 976,
"item_scraped_count": 925,
"log_count/INFO": 21,
"log_count/WARNING": 1,
"scheduler/dequeued": 992,
"scheduler/dequeued/memory": 992,
"scheduler/enqueued": 1050,
"scheduler/enqueued/memory": 1050
},
"time": {
"elapsed": 360.008903,
"log_interval": 60.0,
"log_interval_real": 60.006694,
"start_time": "2023-08-03 23:24:57",
"utcnow": "2023-08-03 23:30:57"
}
}
This extension logs the following configurable sections:
- ``"delta"`` shows how some numeric stats have changed since the last stats
log message.
The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They
must have ``int`` or ``float`` values.
- ``"stats"`` shows the current value of some stats.
The :setting:`PERIODIC_LOG_STATS` setting determines the target stats.
- ``"time"`` shows detailed timing data.
The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or
not to show this section.
This extension logs data at the start, then on a fixed time interval
configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally
right before the crawl ends.
Example extension configuration:
.. code-block:: python
custom_settings = {
"LOG_LEVEL": "INFO",
"PERIODIC_LOG_STATS": {
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
},
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
"PERIODIC_LOG_TIMING_ENABLED": True,
"EXTENSIONS": {
"scrapy.extensions.periodic_log.PeriodicLog": 0,
},
}
.. setting:: PERIODIC_LOG_DELTA
PERIODIC_LOG_DELTA
""""""""""""""""""
Default: ``None``
* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stat values.
* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - show deltas for stats with names containing any configured substring.
* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - show deltas for all stats with names not containing any configured substring.
.. setting:: PERIODIC_LOG_STATS
PERIODIC_LOG_STATS
""""""""""""""""""
Default: ``None``
* ``"PERIODIC_LOG_STATS": True`` - show the current value of all stats.
* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - show current values for stats with names containing any configured substring.
* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - show current values for all stats with names not containing any configured substring.
.. setting:: PERIODIC_LOG_TIMING_ENABLED
PERIODIC_LOG_TIMING_ENABLED
"""""""""""""""""""""""""""
Default: ``False``
``True`` enables logging of timing data (i.e. the ``"time"`` section).
Debugging extensions
--------------------

117
docs/topics/feed-exports.rst
View File

@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which
allows you to generate feeds with the scraped items, using multiple
serialization formats and storage backends.
This page provides detailed documentation for all feed export features. If you
are looking for a step-by-step guide, check out `Zyte’s export guides`_.
.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data
.. _topics-feed-format:
Serialization formats
@ -101,12 +106,12 @@ The storages backends supported out of the box are:
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-s3` (requires boto3_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
not available. For example, the S3 backend is only available if the boto3_
library is installed.
@ -156,8 +161,8 @@ The feeds are stored in the local filesystem.
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
systems though.
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
Alternatively you can also use a :class:`pathlib.Path` object.
.. _topics-feed-storage-ftp:
@ -175,6 +180,12 @@ FTP supports two different connection modes: `active or passive
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -193,7 +204,7 @@ The feeds are stored on `Amazon S3`_.
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
- Required external libraries: `boto3`_ >= 1.20.0
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@ -204,10 +215,18 @@ passed through the following settings:
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
You can also define a custom ACL, custom endpoint, and region name for exported
feeds using these settings:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
- :setting:`AWS_REGION_NAME`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -236,6 +255,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
@ -290,10 +315,11 @@ class, which is the default value of the ``item_filter`` :ref:`feed option <feed
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
method ``accepts`` and taking ``feed_options`` as an argument.
For instance::
For instance:
.. code-block:: python
class MyCustomFilter:
def __init__(self, feed_options):
self.feed_options = feed_options
@ -487,6 +513,8 @@ as a fallback value if that key is not provided for a specific feed definition:
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
- :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported)
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
.. versionadded:: 2.4.0
@ -515,6 +543,10 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
Use ``utf-8`` if you want UTF-8 for JSON too.
.. versionchanged:: 2.8
The :command:`startproject` command now sets this setting to
``utf-8`` in the generated ``settings.py`` file.
.. setting:: FEED_EXPORT_FIELDS
FEED_EXPORT_FIELDS
@ -547,9 +579,12 @@ to ``.json`` or ``.xml``.
FEED_STORE_EMPTY
----------------
Default: ``False``
Default: ``True``
Whether to export empty feeds (i.e. feeds with no items).
If ``False``, and there are no items to export, no new files are created and
existing files are not modified, even if the :ref:`overwrite feed option
<feed-options>` is enabled.
.. setting:: FEED_STORAGES
@ -590,23 +625,27 @@ For a complete list of available values, access the `Canned ACL`_ section on Ama
FEED_STORAGES_BASE
------------------
Default::
Default:
.. code-block:: python
{
'': 'scrapy.extensions.feedexport.FileFeedStorage',
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
"": "scrapy.extensions.feedexport.FileFeedStorage",
"file": "scrapy.extensions.feedexport.FileFeedStorage",
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
}
A dict containing the built-in feed storage backends supported by Scrapy. You
can disable any of these backends by assigning ``None`` to their URI scheme in
:setting:`FEED_STORAGES`. E.g., to disable the built-in FTP storage backend
(without replacement), place this in your ``settings.py``::
(without replacement), place this in your ``settings.py``:
.. code-block:: python
FEED_STORAGES = {
'ftp': None,
"ftp": None,
}
.. setting:: FEED_EXPORTERS
@ -624,26 +663,30 @@ serialization formats and the values are paths to :ref:`Item exporter
FEED_EXPORTERS_BASE
-------------------
Default::
Default:
.. code-block:: python
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
'marshal': 'scrapy.exporters.MarshalItemExporter',
'pickle': 'scrapy.exporters.PickleItemExporter',
"json": "scrapy.exporters.JsonItemExporter",
"jsonlines": "scrapy.exporters.JsonLinesItemExporter",
"jsonl": "scrapy.exporters.JsonLinesItemExporter",
"jl": "scrapy.exporters.JsonLinesItemExporter",
"csv": "scrapy.exporters.CsvItemExporter",
"xml": "scrapy.exporters.XmlItemExporter",
"marshal": "scrapy.exporters.MarshalItemExporter",
"pickle": "scrapy.exporters.PickleItemExporter",
}
A dict containing the built-in feed exporters supported by Scrapy. You can
disable any of these exporters by assigning ``None`` to their serialization
format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
(without replacement), place this in your ``settings.py``::
(without replacement), place this in your ``settings.py``:
.. code-block:: python
FEED_EXPORTERS = {
'csv': None,
"csv": None,
}
@ -673,7 +716,9 @@ generated:
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
For instance, if your settings include:
.. code-block:: python
FEED_EXPORT_BATCH_ITEM_COUNT = 100
@ -742,16 +787,20 @@ The function signature should be as follows:
For example, to include the :attr:`name <scrapy.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
#. Define the following function somewhere in your project:
.. code-block:: python
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
return {**params, "spider_name": spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:
.. code-block:: python
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
FEED_URI_PARAMS = "myproject.utils.uri_params"
#. Use ``%(spider_name)s`` in your feed URI::
@ -760,6 +809,6 @@ source spider in the feed URI:
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _boto3: https://github.com/boto/boto3
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/

60
docs/topics/item-pipeline.rst
View File

@ -81,19 +81,22 @@ Price validation and dropping items with no prices
Let's take a look at the following hypothetical pipeline that adjusts the
``price`` attribute for those items that do not include VAT
(``price_excludes_vat`` attribute), and drops those items which don't
contain a price::
contain a price:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class PricePipeline:
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter.get('price'):
if adapter.get('price_excludes_vat'):
adapter['price'] = adapter['price'] * self.vat_factor
if adapter.get("price"):
if adapter.get("price_excludes_vat"):
adapter["price"] = adapter["price"] * self.vat_factor
return item
else:
raise DropItem(f"Missing price in {item}")
@ -104,16 +107,18 @@ Write items to a JSON lines file
The following pipeline stores all scraped items (from all spiders) into a
single ``items.jsonl`` file, containing one item per line serialized in JSON
format::
format:
.. code-block:: python
import json
from itemadapter import ItemAdapter
class JsonWriterPipeline:
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jsonl', 'w')
self.file = open("items.jsonl", "w")
def close_spider(self, spider):
self.file.close()
@ -135,14 +140,17 @@ MongoDB address and database name are specified in Scrapy settings;
MongoDB collection is named after item class.
The main point of this example is to show how to use :meth:`from_crawler`
method and how to clean up the resources properly.::
method and how to clean up the resources properly.
.. skip: next
.. code-block:: python
import pymongo
from itemadapter import ItemAdapter
class MongoPipeline:
collection_name = 'scrapy_items'
class MongoPipeline:
collection_name = "scrapy_items"
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
@ -151,8 +159,8 @@ method and how to clean up the resources properly.::
@classmethod
def from_crawler(cls, crawler):
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
mongo_uri=crawler.settings.get("MONGO_URI"),
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
)
def open_spider(self, spider):
@ -183,7 +191,7 @@ render a screenshot of the item URL. After the request response is downloaded,
the item pipeline saves the screenshot to a file and adds the filename to the
item.
::
.. code-block:: python
import hashlib
from pathlib import Path
@ -191,6 +199,7 @@ item.
import scrapy
from itemadapter import ItemAdapter
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.defer import maybe_deferred_to_future
@ -204,8 +213,10 @@ item.
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await maybe_deferred_to_future(
spider.crawler.engine.download(request)
)
if response.status != 200:
# Error happened, return item.
@ -228,23 +239,24 @@ Duplicates filter
A filter that looks for duplicate items, and drops those items that were
already processed. Let's say that our items have a unique id, but our spider
returns multiples items with the same id::
returns multiples items with the same id:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class DuplicatesPipeline:
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
if adapter["id"] in self.ids_seen:
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(adapter['id'])
self.ids_seen.add(adapter["id"])
return item
@ -252,11 +264,13 @@ Activating an Item Pipeline component
=====================================
To activate an Item Pipeline component you must add its class to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
:setting:`ITEM_PIPELINES` setting, like in the following example:
.. code-block:: python
ITEM_PIPELINES = {
'myproject.pipelines.PricePipeline': 300,
'myproject.pipelines.JsonWriterPipeline': 800,
"myproject.pipelines.PricePipeline": 300,
"myproject.pipelines.JsonWriterPipeline": 800,
}
The integer values you assign to classes in this setting determine the

138
docs/topics/items.rst
View File

@ -76,10 +76,13 @@ make it the most feature-complete item type:
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Example::
Example:
.. code-block:: python
from scrapy.item import Item, Field
class CustomItem(Item):
one_field = Field()
another_field = Field()
@ -102,10 +105,13 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
Example::
Example:
.. code-block:: python
from dataclasses import dataclass
@dataclass
class CustomItem:
one_field: str
@ -133,10 +139,13 @@ Additionally, ``attr.s`` items also allow to:
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
Example::
Example:
.. code-block:: python
import attr
@attr.s
class CustomItem:
one_field = attr.ib()
@ -152,10 +161,13 @@ Declaring Item subclasses
-------------------------
Item subclasses are declared using a simple class definition syntax and
:class:`Field` objects. Here is an example::
:class:`Field` objects. Here is an example:
.. code-block:: python
import scrapy
class Product(scrapy.Item):
name = scrapy.Field()
price = scrapy.Field()
@ -222,62 +234,68 @@ notice the API is very similar to the :class:`dict` API.
Creating items
''''''''''''''
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
.. code-block:: pycon
>>> product = Product(name="Desktop PC", price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
Getting field values
''''''''''''''''''''
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
.. code-block:: pycon
>>> product['price']
1000
>>> product["name"]
Desktop PC
>>> product.get("name")
Desktop PC
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product["price"]
1000
>>> product.get('last_updated', 'not set')
not set
>>> product["last_updated"]
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product.get("last_updated", "not set")
not set
>>> product.get('lala', 'unknown field')
'unknown field'
>>> product["lala"] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> 'name' in product # is name field populated?
True
>>> product.get("lala", "unknown field")
'unknown field'
>>> 'last_updated' in product # is last_updated populated?
False
>>> "name" in product # is name field populated?
True
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> "last_updated" in product # is last_updated populated?
False
>>> 'lala' in product.fields # is lala a declared field?
False
>>> "last_updated" in product.fields # is last_updated a declared field?
True
>>> "lala" in product.fields # is lala a declared field?
False
Setting field values
''''''''''''''''''''
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
.. code-block:: pycon
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> product["last_updated"] = "today"
>>> product["last_updated"]
today
>>> product["lala"] = "test" # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Accessing all populated values
@ -285,11 +303,13 @@ Accessing all populated values
To access all populated values, just use the typical :class:`dict` API:
>>> product.keys()
['price', 'name']
.. code-block:: pycon
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
>>> product.keys()
['price', 'name']
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
.. _copying-items:
@ -327,18 +347,20 @@ Other common tasks
Creating dicts from items:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
.. code-block:: pycon
Creating items from dicts:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
Creating items from dicts:
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> Product({"name": "Laptop PC", "price": 1500})
Product(price=1500, name='Laptop PC')
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Extending Item subclasses
@ -347,17 +369,21 @@ Extending Item subclasses
You can extend Items (to add more fields or to change some metadata for some
fields) by declaring a subclass of your original Item.
For example::
For example:
.. code-block:: python
class DiscountedProduct(Product):
discount_percent = scrapy.Field(serializer=str)
discount_expiration_date = scrapy.Field()
You can also extend field metadata by using the previous field metadata and
appending more values, or changing existing values, like this::
appending more values, or changing existing values, like this:
.. code-block:: python
class SpecificProduct(Product):
name = scrapy.Field(Product.fields['name'], serializer=my_serializer)
name = scrapy.Field(Product.fields["name"], serializer=my_serializer)
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
keeping all the previously existing metadata values.

6
docs/topics/jobs.rst
View File

@ -51,11 +51,13 @@ loading that attribute from the job directory, when the spider starts and
stops.
Here's an example of a callback that uses the spider state (other spider code
is omitted for brevity)::
is omitted for brevity):
.. code-block:: python
def parse_item(self, response):
# parse item here
self.state['items_count'] = self.state.get('items_count', 0) + 1
self.state["items_count"] = self.state.get("items_count", 0) + 1
Persistence gotchas
===================

96
docs/topics/leaks.rst
View File

@ -70,13 +70,15 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function::
telnet localhost 6023
>>> prefs()
Live References
.. code-block:: pycon
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
FormRequest 878 oldest: 7s ago
>>> prefs()
Live References
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
FormRequest 878 oldest: 7s ago
As you can see, that report also shows the "age" of the oldest object in each
class. If you're running multiple spiders per process chances are you can
@ -114,7 +116,9 @@ a priori, of course) by using the ``trackref`` tool.
After the crawler is running for a few minutes and we notice its memory usage
has grown a lot, we can enter its telnet console and check the live
references::
references:
.. code-block:: pycon
>>> prefs()
Live References
@ -134,19 +138,23 @@ generating the leaks (passing response references inside requests).
Sometimes extra information about live objects can be helpful.
Let's check the oldest response:
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
.. code-block:: pycon
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest("HtmlResponse")
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
If you want to iterate over all objects, instead of getting the oldest one, you
can use the :func:`scrapy.utils.trackref.iter_all` function:
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
.. code-block:: pycon
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all("HtmlResponse")]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
Too many spiders?
-----------------
@ -157,8 +165,10 @@ For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclasses). For
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. code-block:: pycon
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. module:: scrapy.utils.trackref
:synopsis: Track references of live objects
@ -216,30 +226,32 @@ If you use ``pip``, you can install muppy with the following command::
Here's an example to view all Python objects available in
the heap using muppy:
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
.. code-block:: pycon
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
For more info about muppy, refer to the `muppy documentation`_.

10
docs/topics/link-extractors.rst
View File

@ -18,7 +18,9 @@ through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
variable in your spider, and use it from your spider callbacks:
.. code-block:: python
def parse(self, response):
for link in self.link_extractor.extract_links(response):
@ -132,10 +134,12 @@ LxmlLinkExtractor
.. highlight:: python
You can use the following function in ``process_value``::
You can use the following function in ``process_value``:
.. code-block:: python
def process_value(value):
m = re.search("javascript:goToPage\('(.*?)'", value)
m = re.search(r"javascript:goToPage\('(.*?)'", value)
if m:
return m.group(1)

116
docs/topics/loaders.rst
View File

@ -46,18 +46,21 @@ using a proper processing function.
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
chapter <topics-items>`::
chapter <topics-items>`:
.. code-block:: python
from scrapy.loader import ItemLoader
from myproject.items import Product
def parse(self, response):
l = ItemLoader(item=Product(), response=response)
l.add_xpath('name', '//div[@class="product_name"]')
l.add_xpath('name', '//div[@class="product_title"]')
l.add_xpath('price', '//p[@id="price"]')
l.add_css('stock', 'p#stock')
l.add_value('last_updated', 'today') # you can also use literal values
l.add_xpath("name", '//div[@class="product_name"]')
l.add_xpath("name", '//div[@class="product_title"]')
l.add_xpath("price", '//p[@id="price"]')
l.add_css("stock", "p#stock")
l.add_value("last_updated", "today") # you can also use literal values
return l.load_item()
By quickly looking at that code, we can see the ``name`` field is being
@ -93,11 +96,14 @@ will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
One approach to overcome this is to define items using the
:func:`~dataclasses.field` function, with a ``default`` argument::
:func:`~dataclasses.field` function, with a ``default`` argument:
.. code-block:: python
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: Optional[str] = field(default=None)
@ -122,14 +128,16 @@ processor). The result of the output processor is the final value that gets
assigned to the item.
Let's see an example to illustrate how the input and output processors are
called for a particular field (the same applies for any other field)::
called for a particular field (the same applies for any other field):
.. code-block:: python
l = ItemLoader(Product(), some_selector)
l.add_xpath('name', xpath1) # (1)
l.add_xpath('name', xpath2) # (2)
l.add_css('name', css) # (3)
l.add_value('name', 'test') # (4)
return l.load_item() # (5)
l.add_xpath("name", xpath1) # (1)
l.add_xpath("name", xpath2) # (2)
l.add_css("name", css) # (3)
l.add_value("name", "test") # (4)
return l.load_item() # (5)
So what happens is:
@ -184,13 +192,15 @@ processors <itemloaders:built-in-processors>` built-in for convenience.
Declaring Item Loaders
======================
Item Loaders are declared using a class definition syntax. Here is an example::
Item Loaders are declared using a class definition syntax. Here is an example:
.. code-block:: python
from itemloaders.processors import TakeFirst, MapCompose, Join
from scrapy.loader import ItemLoader
class ProductLoader(ItemLoader):
class ProductLoader(ItemLoader):
default_output_processor = TakeFirst()
name_in = MapCompose(str.title)
@ -215,16 +225,20 @@ As seen in the previous section, input and output processors can be declared in
the Item Loader definition, and it's very common to declare input processors
this way. However, there is one more place where you can specify the input and
output processors to use: in the :ref:`Item Field <topics-items-fields>`
metadata. Here is an example::
metadata. Here is an example:
.. code-block:: python
import scrapy
from itemloaders.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_price(value):
if value.isdigit():
return value
class Product(scrapy.Item):
name = scrapy.Field(
input_processor=MapCompose(remove_tags),
@ -235,12 +249,15 @@ metadata. Here is an example::
output_processor=TakeFirst(),
)
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': 'Welcome to my website', 'price': '1000'}
.. code-block:: pycon
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
>>> il.add_value("price", ["&euro;", "<span>1000</span>"])
>>> il.load_item()
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:
@ -263,10 +280,12 @@ declaring, instantiating or using Item Loader. They are used to modify the
behaviour of the input/output processors.
For example, suppose you have a function ``parse_length`` which receives a text
value and extracts a length from it::
value and extracts a length from it:
.. code-block:: python
def parse_length(text, loader_context):
unit = loader_context.get('unit', 'm')
unit = loader_context.get("unit", "m")
# ... length parsing code goes here ...
return parsed_length
@ -278,22 +297,28 @@ function (``parse_length`` in this case) can thus use them.
There are several ways to modify Item Loader context values:
1. By modifying the currently active Item Loader context
(:attr:`~ItemLoader.context` attribute)::
(:attr:`~ItemLoader.context` attribute):
.. code-block:: python
loader = ItemLoader(product)
loader.context['unit'] = 'cm'
loader.context["unit"] = "cm"
2. On Item Loader instantiation (the keyword arguments of Item Loader
``__init__`` method are stored in the Item Loader context)::
``__init__`` method are stored in the Item Loader context):
loader = ItemLoader(product, unit='cm')
.. code-block:: python
loader = ItemLoader(product, unit="cm")
3. On Item Loader declaration, for those input/output processors that support
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
them::
them:
.. code-block:: python
class ProductLoader(ItemLoader):
length_out = MapCompose(parse_length, unit='cm')
length_out = MapCompose(parse_length, unit="cm")
ItemLoader objects
@ -323,25 +348,29 @@ Example::
Without nested loaders, you need to specify the full xpath (or css) for each value
that you wish to extract.
Example::
Example:
.. code-block:: python
loader = ItemLoader(item=Item())
# load stuff not in the footer
loader.add_xpath('social', '//footer/a[@class = "social"]/@href')
loader.add_xpath('email', '//footer/a[@class = "email"]/@href')
loader.add_xpath("social", '//footer/a[@class = "social"]/@href')
loader.add_xpath("email", '//footer/a[@class = "email"]/@href')
loader.load_item()
Instead, you can create a nested loader with the footer selector and add values
relative to the footer. The functionality is the same but you avoid repeating
the footer selector.
Example::
Example:
.. code-block:: python
loader = ItemLoader(item=Item())
# load stuff not in the footer
footer_loader = loader.nested_xpath('//footer')
footer_loader.add_xpath('social', 'a[@class = "social"]/@href')
footer_loader.add_xpath('email', 'a[@class = "email"]/@href')
footer_loader = loader.nested_xpath("//footer")
footer_loader.add_xpath("social", 'a[@class = "social"]/@href')
footer_loader.add_xpath("email", 'a[@class = "email"]/@href')
# no need to call footer_loader.load_item()
loader.load_item()
@ -370,25 +399,32 @@ three dashes (e.g. ``---Plasma TV---``) and you don't want to end up scraping
those dashes in the final product names.
Here's how you can remove those dashes by reusing and extending the default
Product Item Loader (``ProductLoader``)::
Product Item Loader (``ProductLoader``):
.. code-block:: python
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
def strip_dashes(x):
return x.strip('-')
return x.strip("-")
class SiteSpecificLoader(ProductLoader):
name_in = MapCompose(strip_dashes, ProductLoader.name_in)
Another case where extending Item Loaders can be very helpful is when you have
multiple source formats, for example XML and HTML. In the XML version you may
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
want to remove ``CDATA`` occurrences. Here's an example of how to do it:
.. code-block:: python
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
from myproject.utils.xml import remove_cdata
class XmlProductLoader(ProductLoader):
name_in = MapCompose(remove_cdata, ProductLoader.name_in)

111
docs/topics/logging.rst
View File

@ -39,16 +39,22 @@ How to log messages
===================
Here's a quick example of how to log a message using the ``logging.WARNING``
level::
level:
.. code-block:: python
import logging
logging.warning("This is a warning")
There are shortcuts for issuing log messages on any of the standard 5 levels,
and there's also a general ``logging.log`` method which takes a given level as
argument. If needed, the last example could be rewritten as::
argument. If needed, the last example could be rewritten as:
.. code-block:: python
import logging
logging.log(logging.WARNING, "This is a warning")
On top of that, you can create different "loggers" to encapsulate messages. (For
@ -59,24 +65,33 @@ constructions.
The previous examples use the root logger behind the scenes, which is a top level
logger where all messages are propagated to (unless otherwise specified). Using
``logging`` helpers is merely a shortcut for getting the root logger
explicitly, so this is also an equivalent of the last snippets::
explicitly, so this is also an equivalent of the last snippets:
.. code-block:: python
import logging
logger = logging.getLogger()
logger.warning("This is a warning")
You can use a different logger just by getting its name with the
``logging.getLogger`` function::
``logging.getLogger`` function:
.. code-block:: python
import logging
logger = logging.getLogger('mycustomlogger')
logger = logging.getLogger("mycustomlogger")
logger.warning("This is a warning")
Finally, you can ensure having a custom logger for any module you're working on
by using the ``__name__`` variable, which is populated with current module's
path::
path:
.. code-block:: python
import logging
logger = logging.getLogger(__name__)
logger.warning("This is a warning")
@ -94,33 +109,37 @@ Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
instance, which can be accessed and used like this::
instance, which can be accessed and used like this:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapy.org']
class MySpider(scrapy.Spider):
name = "myspider"
start_urls = ["https://scrapy.org"]
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
self.logger.info("Parse function called on %s", response.url)
That logger is created using the Spider's name, but you can use any custom
Python logger you want. For example::
Python logger you want. For example:
.. code-block:: python
import logging
import scrapy
logger = logging.getLogger('mycustomlogger')
logger = logging.getLogger("mycustomlogger")
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapy.org']
name = "myspider"
start_urls = ["https://scrapy.org"]
def parse(self, response):
logger.info('Parse function called on %s', response.url)
logger.info("Parse function called on %s", response.url)
.. _topics-logging-configuration:
@ -229,7 +248,9 @@ the crawl.
Next, we can see that the message has INFO level. To hide it
we should set logging level for ``scrapy.spidermiddlewares.httperror``
higher than INFO; next level after INFO is WARNING. It could be done
e.g. in the spider's ``__init__`` method::
e.g. in the spider's ``__init__`` method:
.. code-block:: python
import logging
import scrapy
@ -238,7 +259,7 @@ e.g. in the spider's ``__init__`` method::
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('scrapy.spidermiddlewares.httperror')
logger = logging.getLogger("scrapy.spidermiddlewares.httperror")
logger.setLevel(logging.WARNING)
super().__init__(*args, **kwargs)
@ -249,43 +270,53 @@ You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
filter out unwanted messages:
.. code-block:: python
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
match = re.search(r"\d{3} [Ee]rror, retrying", record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
(middlewares, spider, etc.):
import logging
import scrapy
.. code-block:: python
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
and hide it without affecting other loggers:
.. code-block:: python
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger = logging.getLogger("my_logger")
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================
@ -306,14 +337,14 @@ scrapy.utils.log module
so it is recommended to only use :func:`logging.basicConfig` together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
This is an example on how to redirect ``INFO`` or higher messages to a file:
.. code-block:: python
import logging
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',
level=logging.INFO
filename="log.txt", format="%(levelname)s: %(message)s", level=logging.INFO
)
Refer to :ref:`run-from-script` for more details about using Scrapy this

181
docs/topics/media-pipeline.rst
View File

@ -87,13 +87,17 @@ Enabling your Media Pipeline
To enable your media pipeline you must first add it to your project
:setting:`ITEM_PIPELINES` setting.
For Images Pipeline, use::
For Images Pipeline, use:
ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}
.. code-block:: python
For Files Pipeline, use::
ITEM_PIPELINES = {"scrapy.pipelines.images.ImagesPipeline": 1}
ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
For Files Pipeline, use:
.. code-block:: python
ITEM_PIPELINES = {"scrapy.pipelines.files.FilesPipeline": 1}
.. note::
You can also use both the Files and Images Pipeline at the same time.
@ -103,13 +107,17 @@ Then, configure the target storage setting to a valid value that will be used
for storing the downloaded images. Otherwise the pipeline will remain disabled,
even if you include it in the :setting:`ITEM_PIPELINES` setting.
For the Files Pipeline, set the :setting:`FILES_STORE` setting::
For the Files Pipeline, set the :setting:`FILES_STORE` setting:
FILES_STORE = '/path/to/valid/dir'
.. code-block:: python
For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
FILES_STORE = "/path/to/valid/dir"
IMAGES_STORE = '/path/to/valid/dir'
For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:
.. code-block:: python
IMAGES_STORE = "/path/to/valid/dir"
.. _topics-file-naming:
@ -157,10 +165,11 @@ By overriding ``file_path`` like this:
import hashlib
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
image_perspective = request.url.split('/')[-2]
image_filename = f'{image_url_hash}_{image_perspective}.jpg'
image_perspective = request.url.split("/")[-2]
image_filename = f"{image_url_hash}_{image_perspective}.jpg"
return image_filename
@ -233,30 +242,38 @@ If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
For example, this is a valid :setting:`IMAGES_STORE` value:
IMAGES_STORE = 's3://bucket/images'
.. code-block:: python
IMAGES_STORE = "s3://bucket/images"
You can modify the Access Control List (ACL) policy used for the stored files,
which is defined by the :setting:`FILES_STORE_S3_ACL` and
:setting:`IMAGES_STORE_S3_ACL` settings. By default, the ACL is set to
``private``. To make the files publicly available use the ``public-read``
policy::
policy:
IMAGES_STORE_S3_ACL = 'public-read'
.. code-block:: python
IMAGES_STORE_S3_ACL = "public-read"
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
settings:
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
.. code-block:: python
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
AWS_ENDPOINT_URL = "http://minio.example.com:9000"
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:
.. code-block:: python
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
@ -277,10 +294,12 @@ bucket. Scrapy will automatically upload the files to the bucket. (requires `goo
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:
IMAGES_STORE = 'gs://bucket/images/'
GCS_PROJECT_ID = 'project_id'
.. code-block:: python
IMAGES_STORE = "gs://bucket/images/"
GCS_PROJECT_ID = "project_id"
For information about authentication, see this `documentation`_.
@ -291,9 +310,11 @@ which is defined by the :setting:`FILES_STORE_GCS_ACL` and
:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
To make the files publicly available use the ``publicRead``
policy::
policy:
IMAGES_STORE_GCS_ACL = 'publicRead'
.. code-block:: python
IMAGES_STORE_GCS_ACL = "publicRead"
For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
@ -318,10 +339,13 @@ respectively), the pipeline will put the results under the respective field
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.Item` class::
``images`` field. For instance, using the :class:`~scrapy.Item` class:
.. code-block:: python
import scrapy
class MyItem(scrapy.Item):
# ... other item fields ...
image_urls = scrapy.Field()
@ -331,16 +355,20 @@ If you want to use another field name for the URLs key or for the results key,
it is also possible to override it.
For the Files Pipeline, set :setting:`FILES_URLS_FIELD` and/or
:setting:`FILES_RESULT_FIELD` settings::
:setting:`FILES_RESULT_FIELD` settings:
FILES_URLS_FIELD = 'field_name_for_your_files_urls'
FILES_RESULT_FIELD = 'field_name_for_your_processed_files'
.. code-block:: python
FILES_URLS_FIELD = "field_name_for_your_files_urls"
FILES_RESULT_FIELD = "field_name_for_your_processed_files"
For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
:setting:`IMAGES_RESULT_FIELD` settings::
:setting:`IMAGES_RESULT_FIELD` settings:
IMAGES_URLS_FIELD = 'field_name_for_your_images_urls'
IMAGES_RESULT_FIELD = 'field_name_for_your_processed_images'
.. code-block:: python
IMAGES_URLS_FIELD = "field_name_for_your_images_urls"
IMAGES_RESULT_FIELD = "field_name_for_your_processed_images"
If you need something more complex and want to override the custom pipeline
behaviour, see :ref:`topics-media-pipeline-override`.
@ -366,7 +394,9 @@ File expiration
The Image Pipeline avoids downloading files that were downloaded recently. To
adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or
:setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which
specifies the delay in number of days::
specifies the delay in number of days:
.. code-block:: python
# 120 days of delay for files expiration
FILES_EXPIRES = 120
@ -400,11 +430,13 @@ images.
In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
where the keys are the thumbnail names and the values are their dimensions.
For example::
For example:
.. code-block:: python
IMAGES_THUMBS = {
'small': (50, 50),
'big': (270, 270),
"small": (50, 50),
"big": (270, 270),
}
When you use this feature, the Images Pipeline will create thumbnails of the
@ -495,17 +527,19 @@ See here the methods that you can override in your custom Files Pipeline:
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
original filenames (e.g. ``files/foo.png``):
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
class MyFilesPipeline(FilesPipeline):
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + PurePosixPath(urlparse(request.url).path).name
return "files/" + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -521,13 +555,16 @@ See here the methods that you can override in your custom Files Pipeline:
As seen on the workflow, the pipeline will get the URLs of the images to
download from the item. In order to do this, you can override the
:meth:`~get_media_requests` method and return a Request for each
file URL::
file URL:
.. code-block:: python
from itemadapter import ItemAdapter
def get_media_requests(self, item, info):
adapter = ItemAdapter(item)
for file_url in adapter['file_urls']:
for file_url in adapter["file_urls"]:
yield scrapy.Request(file_url)
Those requests will be processed by the pipeline and, when they have finished
@ -567,15 +604,26 @@ See here the methods that you can override in your custom Files Pipeline:
guaranteed to retain the same order of the requests returned from the
:meth:`~get_media_requests` method.
Here's a typical value of the ``results`` argument::
Here's a typical value of the ``results`` argument:
[(True,
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
'url': 'http://www.example.com/files/product1.pdf',
'status': 'downloaded'}),
(False,
Failure(...))]
.. invisible-code-block: python
from twisted.python.failure import Failure
.. code-block:: python
[
(
True,
{
"checksum": "2b00042f7481c7b056c4b410d28f33cf",
"path": "full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg",
"url": "http://www.example.com/files/product1.pdf",
"status": "downloaded",
},
),
(False, Failure(...)),
]
By default the :meth:`get_media_requests` method returns ``None`` which
means there are no files to download for the item.
@ -592,17 +640,20 @@ See here the methods that you can override in your custom Files Pipeline:
Here is an example of the :meth:`~item_completed` method where we
store the downloaded file paths (passed in results) in the ``file_paths``
item field, and we drop the item if it doesn't contain any files::
item field, and we drop the item if it doesn't contain any files:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
def item_completed(self, results, item, info):
file_paths = [x['path'] for ok, x in results if ok]
file_paths = [x["path"] for ok, x in results if ok]
if not file_paths:
raise DropItem("Item contains no files")
adapter = ItemAdapter(item)
adapter['file_paths'] = file_paths
adapter["file_paths"] = file_paths
return item
By default, the :meth:`item_completed` method returns the item.
@ -634,17 +685,19 @@ See here the methods that you can override in your custom Images Pipeline:
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
original filenames (e.g. ``files/foo.png``):
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + PurePosixPath(urlparse(request.url).path).name
return "files/" + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -700,33 +753,35 @@ Custom Images pipeline example
==============================
Here is a full example of the Images Pipeline whose methods are exemplified
above::
above:
.. code-block:: python
import scrapy
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
class MyImagesPipeline(ImagesPipeline):
def get_media_requests(self, item, info):
for image_url in item['image_urls']:
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def item_completed(self, results, item, info):
image_paths = [x['path'] for ok, x in results if ok]
image_paths = [x["path"] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
adapter = ItemAdapter(item)
adapter['image_paths'] = image_paths
adapter["image_paths"] = image_paths
return item
To enable your custom media pipeline component you must add its class import path to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
:setting:`ITEM_PIPELINES` setting, like in the following example:
ITEM_PIPELINES = {
'myproject.pipelines.MyImagesPipeline': 300
}
.. code-block:: python
ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5

66
docs/topics/practices.rst
View File

@ -7,6 +7,8 @@ Common Practices
This section documents common practices when using Scrapy. These are things
that cover many topics and don't often fall into any other specific section.
.. skip: start
.. _run-from-script:
Run Scrapy from a script
@ -25,23 +27,27 @@ the one used by all Scrapy commands.
Here's an example showing how to run a single spider with it.
::
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerProcess
class MySpider(scrapy.Spider):
# Your spider definition
...
process = CrawlerProcess(settings={
"FEEDS": {
"items.json": {"format": "json"},
},
})
process = CrawlerProcess(
settings={
"FEEDS": {
"items.json": {"format": "json"},
},
}
)
process.crawl(MySpider)
process.start() # the script will block here until the crawling is finished
process.start() # the script will block here until the crawling is finished
Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
documentation to get acquainted with its usage details.
@ -55,7 +61,7 @@ instance with your project settings.
What follows is a working example of how to do that, using the `testspiders`_
project as example.
::
.. code-block:: python
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
@ -63,8 +69,8 @@ project as example.
process = CrawlerProcess(get_project_settings())
# 'followall' is the name of one of the spiders of the project.
process.crawl('followall', domain='scrapy.org')
process.start() # the script will block here until the crawling is finished
process.crawl("followall", domain="scrapy.org")
process.start() # the script will block here until the crawling is finished
There's another Scrapy utility that provides more control over the crawling
process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper
@ -84,23 +90,25 @@ returned by the :meth:`CrawlerRunner.crawl
Here's an example of its usage, along with a callback to manually stop the
reactor after ``MySpider`` has finished running.
::
.. code-block:: python
from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class MySpider(scrapy.Spider):
# Your spider definition
...
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
runner = CrawlerRunner()
d = runner.crawl(MySpider)
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
reactor.run() # the script will block here until the crawling is finished
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
@ -115,29 +123,32 @@ the :ref:`internal API <topics-api>`.
Here is an example that runs multiple spiders simultaneously:
::
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
settings = get_project_settings()
process = CrawlerProcess(settings)
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start() # the script will block here until all crawling jobs are finished
process.start() # the script will block here until all crawling jobs are finished
Same example using :class:`~scrapy.crawler.CrawlerRunner`:
::
.. code-block:: python
import scrapy
from twisted.internet import reactor
@ -145,14 +156,17 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
configure_logging()
settings = get_project_settings()
runner = CrawlerRunner(settings)
@ -161,37 +175,42 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
d = runner.join()
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until all crawling jobs are finished
reactor.run() # the script will block here until all crawling jobs are finished
Same example but running the spiders sequentially by chaining the deferreds:
::
.. code-block:: python
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
settings = get_project_settings()
configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
def crawl():
yield runner.crawl(MySpider1)
yield runner.crawl(MySpider2)
reactor.stop()
crawl()
reactor.run() # the script will block here until the last crawl call is finished
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some
@ -214,6 +233,8 @@ different for different settings:
.. seealso:: :ref:`run-from-script`.
.. skip: end
.. _distributed-crawls:
Distributed crawls
@ -267,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
`Zyte Smart Proxy Manager`_
* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__
If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
@ -280,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _Common Crawl: https://commoncrawl.org/
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html

300
docs/topics/request-response.rst
View File

@ -32,11 +32,20 @@ Request objects
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
request (once it's downloaded) as its first parameter.
In addition to a function, the following values are supported:
- ``None`` (default), which indicates that the spider's
:meth:`~scrapy.Spider.parse` method must be used.
- :func:`~scrapy.http.request.NO_CALLBACK`
For more information, see
:ref:`topics-request-response-ref-request-callback-arguments`.
.. note:: If exceptions are raised during processing, ``errback`` is
called instead.
:type callback: collections.abc.Callable
@ -67,18 +76,34 @@ Request objects
:param cookies: the request cookies. These can be sent in two forms.
1. Using a dict::
.. invisible-code-block: python
request_with_cookies = Request(url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'})
from scrapy.http import Request
2. Using a list of dicts::
1. Using a dict:
request_with_cookies = Request(url="http://www.example.com",
cookies=[{'name': 'currency',
'value': 'USD',
'domain': 'example.com',
'path': '/currency'}])
.. code-block:: python
request_with_cookies = Request(
url="http://www.example.com",
cookies={"currency": "USD", "country": "UY"},
)
2. Using a list of dicts:
.. code-block:: python
request_with_cookies = Request(
url="http://www.example.com",
cookies=[
{
"name": "currency",
"value": "USD",
"domain": "example.com",
"path": "/currency",
},
],
)
The latter form allows for customizing the ``domain`` and ``path``
attributes of the cookie. This is only useful if the cookies are saved
@ -90,18 +115,9 @@ Request objects
cookies for that domain and will be sent again in future requests.
That's the typical behaviour of any regular web browser.
To create a request that does not send stored cookies and does not
store received cookies, set the ``dont_merge_cookies`` key to ``True``
in :attr:`request.meta <scrapy.Request.meta>`.
Example of a request that sends manually-defined cookies and ignores
cookie storage::
Request(
url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'},
meta={'dont_merge_cookies': True},
)
Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in
:attr:`request.meta <scrapy.Request.meta>` causes custom cookies to be
ignored.
For more info see :ref:`cookies-mw`.
@ -177,18 +193,47 @@ Request objects
:meth:`replace`.
.. attribute:: Request.meta
:value: {}
A dict that contains arbitrary metadata for this request. This dict is
empty for new Requests, and is usually populated by different Scrapy
components (extensions, middlewares, etc). So the data contained in this
dict depends on the extensions you have enabled.
A dictionary of arbitrary metadata for the request.
See :ref:`topics-request-meta` for a list of special meta keys
recognized by Scrapy.
You may extend request metadata as you see fit.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.meta`` attribute.
Request metadata can also be accessed through the
:attr:`~scrapy.http.Response.meta` attribute of a response.
To pass data from one spider callback to another, consider using
:attr:`cb_kwargs` instead. However, request metadata may be the right
choice in certain scenarios, such as to maintain some debugging data
across all follow-up requests (e.g. the source URL).
A common use of request metadata is to define request-specific
parameters for Scrapy components (extensions, middlewares, etc.). For
example, if you set ``dont_retry`` to ``True``,
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never
retry that request, even if it fails. See :ref:`topics-request-meta`.
You may also use request metadata in your custom Scrapy components, for
example, to keep request state information relevant to your component.
For example,
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the
``retry_times`` metadata key to keep track of how many times a request
has been retried so far.
Copying all the metadata of a previous request into a new, follow-up
request in a spider callback is a bad practice, because request
metadata may include metadata set by Scrapy components that is not
meant to be copied into other requests. For example, copying the
``retry_times`` metadata key into follow-up requests can lower the
amount of retries allowed for those follow-up requests.
You should only copy all request metadata from one request to another
if the new request is meant to replace the old request, as is often the
case when returning a request from a :ref:`downloader middleware
<topics-downloader-middleware>` method.
Also mind that the :meth:`copy` and :meth:`replace` request methods
:doc:`shallow-copy <library/copy>` request metadata.
.. attribute:: Request.cb_kwargs
@ -228,6 +273,8 @@ Request objects
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.http.request.NO_CALLBACK
.. autofunction:: scrapy.utils.request.request_from_dict
@ -240,11 +287,15 @@ The callback of a request is a function that will be called when the response
of that request is downloaded. The callback function will be called with the
downloaded :class:`Response` object as its first argument.
Example::
Example:
.. code-block:: python
def parse_page1(self, response):
return scrapy.Request("http://www.example.com/some_page.html",
callback=self.parse_page2)
return scrapy.Request(
"http://www.example.com/some_page.html", callback=self.parse_page2
)
def parse_page2(self, response):
# this would log http://www.example.com/some_page.html
@ -255,15 +306,18 @@ functions so you can receive the arguments later, in the second callback.
The following example shows how to achieve this by using the
:attr:`Request.cb_kwargs` attribute:
::
.. code-block:: python
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url))
request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url),
)
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
yield request
def parse_page2(self, response, main_url, foo):
yield dict(
main_url=main_url,
@ -289,7 +343,9 @@ It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
errors if needed:
.. code-block:: python
import scrapy
@ -297,24 +353,28 @@ errors if needed::
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
class ErrbackSpider(scrapy.Spider):
name = "errback_example"
start_urls = [
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
]
def start_requests(self):
for u in self.start_urls:
yield scrapy.Request(u, callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True)
yield scrapy.Request(
u,
callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True,
)
def parse_httpbin(self, response):
self.logger.info('Got successful response from {}'.format(response.url))
self.logger.info("Got successful response from {}".format(response.url))
# do something useful here...
def errback_httpbin(self, failure):
@ -328,16 +388,16 @@ errors if needed::
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
self.logger.error('HttpError on %s', response.url)
self.logger.error("HttpError on %s", response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error('DNSLookupError on %s', request.url)
self.logger.error("DNSLookupError on %s", request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
self.logger.error("TimeoutError on %s", request.url)
.. _errback-cb_kwargs:
@ -348,21 +408,27 @@ Accessing additional data in errback functions
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``::
achieve this by using ``Failure.request.cb_kwargs``:
.. code-block:: python
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url))
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url),
)
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs['main_url'],
main_url=failure.request.cb_kwargs["main_url"],
)
@ -509,18 +575,20 @@ in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
For example, to take the value of a request header named ``X-ID`` into
account::
account:
.. code-block:: python
# my_project/settings.py
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
REQUEST_FINGERPRINTER_CLASS = "my_project.utils.RequestFingerprinter"
# my_project/utils.py
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
class RequestFingerprinter:
def fingerprint(self, request):
return fingerprint(request, include_headers=['X-ID'])
return fingerprint(request, include_headers=["X-ID"])
You can also write your own fingerprinting logic from scratch.
@ -536,15 +604,17 @@ you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
references to them in your cache dictionary.
For example, to take into account only the URL of a request, without any prior
URL canonicalization or taking the request method or body into account::
URL canonicalization or taking the request method or body into account:
.. code-block:: python
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
class RequestFingerprinter:
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
@ -558,21 +628,25 @@ If you need to be able to override the request fingerprinting for arbitrary
requests from your spider callbacks, you may implement a request fingerprinter
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
when available, and then falls back to
:func:`scrapy.utils.request.fingerprint`. For example::
:func:`scrapy.utils.request.fingerprint`. For example:
.. code-block:: python
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
class RequestFingerprinter:
def fingerprint(self, request):
if 'fingerprint' in request.meta:
return request.meta['fingerprint']
if "fingerprint" in request.meta:
return request.meta["fingerprint"]
return fingerprint(request)
If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
without using the deprecated ``'2.6'`` value of the
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
request fingerprinter::
request fingerprinter:
.. code-block:: python
from hashlib import sha1
from weakref import WeakKeyDictionary
@ -580,8 +654,8 @@ request fingerprinter::
from scrapy.utils.python import to_bytes
from w3lib.url import canonicalize_url
class RequestFingerprinter:
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
@ -589,7 +663,7 @@ request fingerprinter::
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b'')
fp.update(request.body or b"")
self.cache[request] = fp.digest()
return self.cache[request]
@ -718,7 +792,9 @@ Stopping the download of a Response
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signals will stop the download of a given response. See the following example::
signals will stop the download of a given response. See the following example:
.. code-block:: python
import scrapy
@ -730,7 +806,9 @@ signals will stop the download of a given response. See the following example::
@classmethod
def from_crawler(cls, crawler):
spider = super().from_crawler(crawler)
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
crawler.signals.connect(
spider.on_bytes_received, signal=scrapy.signals.bytes_received
)
return spider
def parse(self, response):
@ -859,11 +937,18 @@ Using FormRequest to send data via HTTP POST
If you want to simulate a HTML Form POST in your spider and send a couple of
key-value fields, you can return a :class:`FormRequest` object (from your
spider) like this::
spider) like this:
return [FormRequest(url="http://www.example.com/post/action",
formdata={'name': 'John Doe', 'age': '27'},
callback=self.after_post)]
.. skip: next
.. code-block:: python
return [
FormRequest(
url="http://www.example.com/post/action",
formdata={"name": "John Doe", "age": "27"},
callback=self.after_post,
)
]
.. _topics-request-response-ref-request-userlogin:
@ -875,25 +960,28 @@ type="hidden">`` elements, such as session related data or authentication
tokens (for login pages). When scraping, you'll want these fields to be
automatically pre-populated and only override a couple of them, such as the
user name and password. You can use the :meth:`FormRequest.from_response`
method for this job. Here's an example spider which uses it::
method for this job. Here's an example spider which uses it:
.. code-block:: python
import scrapy
def authentication_failed(response):
# TODO: Check the contents of the response and return True if it failed
# or False if it succeeded.
pass
class LoginSpider(scrapy.Spider):
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
name = "example.com"
start_urls = ["http://www.example.com/users/login.php"]
def parse(self, response):
return scrapy.FormRequest.from_response(
response,
formdata={'username': 'john', 'password': 'secret'},
callback=self.after_login
formdata={"username": "john", "password": "secret"},
callback=self.after_login,
)
def after_login(self, response):
@ -933,13 +1021,16 @@ dealing with JSON requests.
JsonRequest usage example
-------------------------
Sending a JSON POST request with a JSON payload::
Sending a JSON POST request with a JSON payload:
.. skip: next
.. code-block:: python
data = {
'name1': 'value1',
'name2': 'value2',
"name1": "value1",
"name2": "value2",
}
yield JsonRequest(url='http://www.example.com/post/action', data=data)
yield JsonRequest(url="http://www.example.com/post/action", data=data)
Response objects
@ -1030,9 +1121,10 @@ Response objects
through all :ref:`Downloader Middlewares <topics-downloader-middleware>`.
In particular, this means that:
- HTTP redirections will cause the original request (to the URL before
redirection) to be assigned to the redirected response (with the final
URL after redirection).
- HTTP redirections will create a new request from the request before
redirection. It has the majority of the same metadata and original
request attributes and gets assigned to the redirected response
instead of the propagation of the original request.
- Response.request.url doesn't always equal Response.url
@ -1174,8 +1266,10 @@ TextResponse objects
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. code-block:: pycon
>>> str(b"body")
"b'body'"
.. attribute:: TextResponse.encoding
@ -1208,6 +1302,12 @@ TextResponse objects
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
.. method:: TextResponse.jmespath(query)
A shortcut to ``TextResponse.selector.jmespath(query)``::
response.jmespath('object.[*]')
.. method:: TextResponse.xpath(query)
A shortcut to ``TextResponse.selector.xpath(query)``::
@ -1259,3 +1359,13 @@ XmlResponse objects
line. See :attr:`TextResponse.encoding`.
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
JsonResponse objects
--------------------
.. class:: JsonResponse(url[, ...])
The :class:`JsonResponse` class is a subclass of :class:`TextResponse`
that is used when the response has a `JSON MIME type
<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`
header.

664
docs/topics/selectors.rst
View File

File diff suppressed because it is too large Load Diff

335
docs/topics/settings.rst
View File

@ -40,8 +40,9 @@ precedence:
1. Command line options (most precedence)
2. Settings per-spider
3. Project settings module
4. Default settings per-command
5. Default global settings (less precedence)
4. Settings set by add-ons
5. Default settings per-command
6. Default global settings (less precedence)
The population of these settings sources is taken care of internally, but a
manual handling is possible using API calls. See the
@ -66,16 +67,60 @@ Example::
----------------------
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
own settings that will take precedence and override the project ones. They can
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
own settings that will take precedence and override the project ones. One way
to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
custom_settings = {
'SOME_SETTING': 'some value',
"SOME_SETTING": "some value",
}
It's often better to implement :meth:`~scrapy.Spider.update_settings` instead,
and settings set there should use the "spider" priority explicitly:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
@classmethod
def update_settings(cls, settings):
super().update_settings(settings)
settings.set("SOME_SETTING", "some value", priority="spider")
.. versionadded:: 2.11
It's also possible to modify the settings in the
:meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider
arguments <spiderargs>` or other logic:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
if "some_argument" in kwargs:
spider.settings.set(
"SOME_SETTING", kwargs["some_argument"], priority="spider"
)
return spider
3. Project settings module
--------------------------
@ -84,7 +129,13 @@ project, it's where most of your custom settings will be populated. For a
standard Scrapy project, this means you'll be adding or changing the settings
in the ``settings.py`` file created for your project.
4. Default settings per-command
4. Settings set by add-ons
--------------------------
:ref:`Add-ons <topics-addons>` can modify settings. They should do this with
this priority, though this is not enforced.
5. Default settings per-command
-------------------------------
Each :doc:`Scrapy tool </topics/commands>` command can have its own default
@ -92,7 +143,7 @@ settings, which override the global default settings. Those custom command
settings are specified in the ``default_settings`` attribute of the command
class.
5. Default global settings
6. Default global settings
--------------------------
The global defaults are located in the ``scrapy.settings.default_settings``
@ -115,14 +166,18 @@ class or a function, there are two different ways you can specify that object:
- As the object itself
For example::
For example:
.. skip: next
.. code-block:: python
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
"mybot.pipelines.validate.ValidateMyItem": 300,
}
.. note:: Passing non-callable objects is not supported.
@ -133,11 +188,13 @@ How to access settings
.. highlight:: python
In a spider, the settings are available through ``self.settings``::
In a spider, the settings are available through ``self.settings``:
.. code-block:: python
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://example.com']
name = "myspider"
start_urls = ["http://example.com"]
def parse(self, response):
print(f"Existing settings: {self.settings.attributes.keys()}")
@ -150,7 +207,9 @@ In a spider, the settings are available through ``self.settings``::
Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
extensions, middlewares and item pipelines::
extensions, middlewares and item pipelines:
.. code-block:: python
class MyExtension:
def __init__(self, log_is_enabled=False):
@ -160,7 +219,7 @@ extensions, middlewares and item pipelines::
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
return cls(settings.getbool('LOG_ENABLED'))
return cls(settings.getbool("LOG_ENABLED"))
The settings object can be used like a dict (e.g.,
``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting
@ -188,6 +247,16 @@ to any particular component. In that case the module of that component will be
shown, typically an extension, middleware or pipeline. It also means that the
component must be enabled in order for the setting to have any effect.
.. setting:: ADDONS
ADDONS
------
Default: ``{}``
A dict containing paths to the add-ons enabled in your project and their
priorities. For more information, see :ref:`topics-addons`.
.. setting:: AWS_ACCESS_KEY_ID
AWS_ACCESS_KEY_ID
@ -365,11 +434,13 @@ Scrapy shell <topics-shell>`.
DEFAULT_REQUEST_HEADERS
-----------------------
Default::
Default:
.. code-block:: python
{
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en",
}
The default headers used for Scrapy HTTP Requests. They're populated in the
@ -404,9 +475,12 @@ Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
a :class:`~scrapy.Request` based on its depth.
The priority of a request is adjusted as follows::
The priority of a request is adjusted as follows:
request.priority = request.priority - ( depth * DEPTH_PRIORITY )
.. skip: next
.. code-block:: python
request.priority = request.priority - (depth * DEPTH_PRIORITY)
As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
priority (BFO), while negative values increase request priority (DFO). See
@ -595,23 +669,25 @@ orders. For more info see :ref:`topics-downloader-middleware-setting`.
DOWNLOADER_MIDDLEWARES_BASE
---------------------------
Default::
Default:
.. code-block:: python
{
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
"scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
"scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
"scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
"scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560,
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
"scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700,
"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
"scrapy.downloadermiddlewares.stats.DownloaderStats": 850,
"scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900,
}
A dict containing the downloader middlewares enabled by default in Scrapy. Low
@ -636,19 +712,30 @@ DOWNLOAD_DELAY
Default: ``0``
The amount of time (in secs) that the downloader should wait before downloading
consecutive pages from the same website. This can be used to throttle the
crawling speed to avoid hitting servers too hard. Decimal numbers are
supported. Example::
Minimum seconds to wait between 2 consecutive requests to the same domain.
DOWNLOAD_DELAY = 0.25 # 250 ms of delay
Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
servers too hard.
Decimal numbers are supported. For example, to send a maximum of 4 requests
every 10 seconds::
DOWNLOAD_DELAY = 2.5
This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
setting (which is enabled by default). By default, Scrapy doesn't wait a fixed
amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`.
setting, which is enabled by default.
When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
per ip address instead of per domain.
per IP address instead of per domain.
Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
concurrency for that domain is 1. When testing throttling configurations, it
usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first,
and only increase :setting:`DOWNLOAD_DELAY` once
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is
desired.
.. _spider-download_delay-attribute:
@ -656,6 +743,11 @@ per ip address instead of per domain.
This delay can be set per spider using :attr:`download_delay` spider attribute.
It is also possible to change this setting per domain, although it requires
non-trivial code. See the implementation of the :ref:`AutoThrottle
<topics-autothrottle>` extension for an example.
.. setting:: DOWNLOAD_HANDLERS
DOWNLOAD_HANDLERS
@ -671,15 +763,17 @@ See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
DOWNLOAD_HANDLERS_BASE
----------------------
Default::
Default:
.. code-block:: python
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
"file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",
"http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
"https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
"s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
}
@ -689,10 +783,12 @@ You should never modify this setting in your project, modify
You can disable any of these download handlers by assigning ``None`` to their
URI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP
handler (without replacement), place this in your ``settings.py``::
handler (without replacement), place this in your ``settings.py``:
.. code-block:: python
DOWNLOAD_HANDLERS = {
'ftp': None,
"ftp": None,
}
.. _http2:
@ -702,10 +798,12 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
#. Update :setting:`DOWNLOAD_HANDLERS` as follows:
.. code-block:: python
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
}
.. warning::
@ -734,6 +832,31 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
.. setting:: DOWNLOAD_SLOTS
DOWNLOAD_SLOTS
----------------
Default: ``{}``
Allows to define concurrency/delay parameters on per slot (domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
"quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
.. note::
For other downloader slots default settings values will be used:
- :setting:`DOWNLOAD_DELAY`: ``delay``
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@ -874,18 +997,20 @@ A dict containing the extensions enabled in your project, and their orders.
EXTENSIONS_BASE
---------------
Default::
Default:
.. code-block:: python
{
'scrapy.extensions.corestats.CoreStats': 0,
'scrapy.extensions.telnet.TelnetConsole': 0,
'scrapy.extensions.memusage.MemoryUsage': 0,
'scrapy.extensions.memdebug.MemoryDebugger': 0,
'scrapy.extensions.closespider.CloseSpider': 0,
'scrapy.extensions.feedexport.FeedExporter': 0,
'scrapy.extensions.logstats.LogStats': 0,
'scrapy.extensions.spiderstate.SpiderState': 0,
'scrapy.extensions.throttle.AutoThrottle': 0,
"scrapy.extensions.corestats.CoreStats": 0,
"scrapy.extensions.telnet.TelnetConsole": 0,
"scrapy.extensions.memusage.MemoryUsage": 0,
"scrapy.extensions.memdebug.MemoryDebugger": 0,
"scrapy.extensions.closespider.CloseSpider": 0,
"scrapy.extensions.feedexport.FeedExporter": 0,
"scrapy.extensions.logstats.LogStats": 0,
"scrapy.extensions.spiderstate.SpiderState": 0,
"scrapy.extensions.throttle.AutoThrottle": 0,
}
A dict containing the extensions available by default in Scrapy, and their
@ -895,7 +1020,6 @@ some of them need to be enabled through a setting.
For more information See the :ref:`extensions user guide <topics-extensions>`
and the :ref:`list of available extensions <topics-extensions-ref>`.
.. setting:: FEED_TEMPDIR
FEED_TEMPDIR
@ -972,11 +1096,13 @@ A dict containing the item pipelines to use, and their orders. Order values are
arbitrary, but it is customary to define them in the 0-1000 range. Lower orders
process before higher orders.
Example::
Example:
.. code-block:: python
ITEM_PIPELINES = {
'mybot.pipelines.validate.ValidateMyItem': 300,
'mybot.pipelines.validate.StoreMyItem': 800,
"mybot.pipelines.validate.ValidateMyItem": 300,
"mybot.pipelines.validate.StoreMyItem": 800,
}
.. setting:: ITEM_PIPELINES_BASE
@ -994,7 +1120,7 @@ modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
JOBDIR
------
Default: ``''``
Default: ``None``
A string indicating the directory for storing the state of a crawl when
:ref:`pausing and resuming crawls <topics-jobs>`.
@ -1044,7 +1170,7 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
:ref:`Python logging documentation <logrecord-attributes>` for the whole
list of available placeholders.
.. setting:: LOG_DATEFORMAT
@ -1401,12 +1527,14 @@ testing spiders. For more info see :ref:`topics-contracts`.
SPIDER_CONTRACTS_BASE
---------------------
Default::
Default:
.. code-block:: python
{
'scrapy.contracts.default.UrlContract' : 1,
'scrapy.contracts.default.ReturnsContract': 2,
'scrapy.contracts.default.ScrapesContract': 3,
"scrapy.contracts.default.UrlContract": 1,
"scrapy.contracts.default.ReturnsContract": 2,
"scrapy.contracts.default.ScrapesContract": 3,
}
A dict containing the Scrapy contracts enabled by default in Scrapy. You should
@ -1415,10 +1543,12 @@ instead. For more info see :ref:`topics-contracts`.
You can disable any of these contracts by assigning ``None`` to their class
path in :setting:`SPIDER_CONTRACTS`. E.g., to disable the built-in
``ScrapesContract``, place this in your ``settings.py``::
``ScrapesContract``, place this in your ``settings.py``:
.. code-block:: python
SPIDER_CONTRACTS = {
'scrapy.contracts.default.ScrapesContract': None,
"scrapy.contracts.default.ScrapesContract": None,
}
.. setting:: SPIDER_LOADER_CLASS
@ -1467,14 +1597,16 @@ orders. For more info see :ref:`topics-spider-middleware-setting`.
SPIDER_MIDDLEWARES_BASE
-----------------------
Default::
Default:
.. code-block:: python
{
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
}
A dict containing the spider middlewares enabled by default in Scrapy, and
@ -1490,9 +1622,11 @@ Default: ``[]``
A list of modules where Scrapy will look for spiders.
Example::
Example:
SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev']
.. code-block:: python
SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"]
.. setting:: STATS_CLASS
@ -1581,60 +1715,65 @@ If a reactor is already installed,
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
In order to use the reactor installed by Scrapy::
In order to use the reactor installed by Scrapy:
.. code-block:: python
import scrapy
from twisted.internet import reactor
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
self.timeout = int(kwargs.pop("timeout", "60"))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['https://quotes.toscrape.com/page/1']
urls = ["https://quotes.toscrape.com/page/1"]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
for quote in response.css("div.quote"):
yield {"text": quote.css("span.text::text").get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
self.crawler.engine.close_spider(self, "timeout")
which raises :exc:`Exception`, becomes::
which raises :exc:`Exception`, becomes:
.. code-block:: python
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
self.timeout = int(kwargs.pop("timeout", "60"))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['https://quotes.toscrape.com/page/1']
urls = ["https://quotes.toscrape.com/page/1"]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
for quote in response.css("div.quote"):
yield {"text": quote.css("span.text::text").get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
self.crawler.engine.close_spider(self, "timeout")
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which

81
docs/topics/shell.rst
View File

@ -191,44 +191,46 @@ all start with the ``[s]`` prefix)::
After that, we can start playing with the objects:
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
.. code-block:: pycon
>>> fetch("https://old.reddit.com/")
>>> response.xpath("//title/text()").get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'
>>> fetch("https://old.reddit.com/")
>>> request = request.replace(method="POST")
>>> response.xpath("//title/text()").get()
'reddit: the front page of the internet'
>>> fetch(request)
>>> request = request.replace(method="POST")
>>> response.status
404
>>> fetch(request)
>>> from pprint import pprint
>>> response.status
404
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
>>> from pprint import pprint
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
.. _topics-shell-inspect-response:
@ -242,7 +244,9 @@ getting there.
This can be achieved by using the ``scrapy.shell.inspect_response`` function.
Here's an example of how you would call it from your spider::
Here's an example of how you would call it from your spider:
.. code-block:: python
import scrapy
@ -259,6 +263,7 @@ Here's an example of how you would call it from your spider::
# We want to inspect one specific response.
if ".org" in response.url:
from scrapy.shell import inspect_response
inspect_response(response, self)
# Rest of parsing code.
@ -276,14 +281,18 @@ When you run the spider, you will get something similar to this::
Then, you can check if the extraction code is working:
>>> response.xpath('//h1[@class="fn"]')
[]
.. code-block:: pycon
>>> response.xpath('//h1[@class="fn"]')
[]
Nope, it doesn't. So you can open the response in your web browser and see if
it's the response you were expecting:
>>> view(response)
True
.. code-block:: pycon
>>> view(response)
True
Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
crawling::

61
docs/topics/signals.rst
View File

@ -16,7 +16,9 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action::
Here is a simple example showing how you can catch signals and perform some action:
.. code-block:: python
from scrapy import signals
from scrapy import Spider
@ -30,17 +32,14 @@ Here is a simple example showing how you can catch signals and perform some acti
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def spider_closed(self, spider):
spider.logger.info('Spider closed: %s', spider.name)
spider.logger.info("Spider closed: %s", spider.name)
def parse(self, response):
pass
@ -56,11 +55,16 @@ you to run asynchronous code that does not block Scrapy. If a signal
handler returns one of these objects, Scrapy waits for that asynchronous
operation to finish.
Let's take an example using :ref:`coroutines <topics-coroutines>`::
Let's take an example using :ref:`coroutines <topics-coroutines>`:
.. code-block:: python
import scrapy
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['https://quotes.toscrape.com/page/1/']
name = "signals"
start_urls = ["https://quotes.toscrape.com/page/1/"]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
@ -71,19 +75,19 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`::
async def item_scraped(self, item):
# Send the scraped item to the server
response = await treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
"http://example.com/post",
json.dumps(item).encode("ascii"),
headers={b"Content-Type": [b"application/json"]},
)
return response
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support
@ -303,6 +307,33 @@ spider_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object
feed_slot_closed
~~~~~~~~~~~~~~~~
.. signal:: feed_slot_closed
.. function:: feed_slot_closed(slot)
Sent when a :ref:`feed exports <topics-feed-exports>` slot is closed.
This signal supports returning deferreds from its handlers.
:param slot: the slot closed
:type slot: scrapy.extensions.feedexport.FeedSlot
feed_exporter_closed
~~~~~~~~~~~~~~~~~~~~
.. signal:: feed_exporter_closed
.. function:: feed_exporter_closed()
Sent when the :ref:`feed exports <topics-feed-exports>` extension is closed,
during the handling of the :signal:`spider_closed` signal by the extension,
after all feed exporting has been handled.
This signal supports returning deferreds from its handlers.
Request signals
---------------

21
docs/topics/spider-middleware.rst
View File

@ -18,10 +18,12 @@ To activate a spider middleware component, add it to the
:setting:`SPIDER_MIDDLEWARES` setting, which is a dict whose keys are the
middleware class path and their values are the middleware orders.
Here's an example::
Here's an example:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'myproject.middlewares.CustomSpiderMiddleware': 543,
"myproject.middlewares.CustomSpiderMiddleware": 543,
}
The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
@ -44,11 +46,13 @@ previous (or subsequent) middleware being applied.
If you want to disable a builtin middleware (the ones defined in
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
value. For example, if you want to disable the off-site middleware::
value. For example, if you want to disable the off-site middleware:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'myproject.middlewares.CustomSpiderMiddleware': 543,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
"myproject.middlewares.CustomSpiderMiddleware": 543,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -261,7 +265,12 @@ specify which response codes the spider is able to handle using the
:setting:`HTTPERROR_ALLOWED_CODES` setting.
For example, if you want your spider to handle 404 responses you can do
this::
this:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class MySpider(CrawlSpider):
handle_httpstatus_list = [404]

310
docs/topics/spiders.rst
View File

@ -99,7 +99,7 @@ scrapy.Spider
.. attribute:: crawler
This attribute is set by the :meth:`from_crawler` class method after
initializating the class, and links to the
initializing the class, and links to the
:class:`~scrapy.crawler.Crawler` object to which this spider instance is
bound.
@ -136,6 +136,21 @@ scrapy.Spider
attributes in the new instance so they can be accessed later inside the
spider's code.
.. versionchanged:: 2.11
The settings in ``crawler.settings`` can now be modified in this
method, which is handy if you want to modify them based on
arguments. As a consequence, these settings aren't the final values
as they can be modified later by e.g. :ref:`add-ons
<topics-addons>`. For the same reason, most of the
:class:`~scrapy.crawler.Crawler` attributes aren't initialized at
this point.
The final settings and the initialized
:class:`~scrapy.crawler.Crawler` attributes are available in the
:meth:`start_requests` method, handlers of the
:signal:`engine_started` signal and later.
:param crawler: crawler to which the spider will be bound
:type crawler: :class:`~scrapy.crawler.Crawler` instance
@ -145,6 +160,46 @@ scrapy.Spider
:param kwargs: keyword arguments passed to the :meth:`__init__` method
:type kwargs: dict
.. classmethod:: update_settings(settings)
The ``update_settings()`` method is used to modify the spider's settings
and is called during initialization of a spider instance.
It takes a :class:`~scrapy.settings.Settings` object as a parameter and
can add or update the spider's configuration values. This method is a
class method, meaning that it is called on the :class:`~scrapy.Spider`
class and allows all instances of the spider to share the same
configuration.
While per-spider settings can be set in
:attr:`~scrapy.Spider.custom_settings`, using ``update_settings()``
allows you to dynamically add, remove or change settings based on other
settings, spider attributes or other factors and use setting priorities
other than ``'spider'``. Also, it's easy to extend ``update_settings()``
in a subclass by overriding it, while doing the same with
:attr:`~scrapy.Spider.custom_settings` can be hard.
For example, suppose a spider needs to modify :setting:`FEEDS`:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
custom_feed = {
"/home/user/documents/items.json": {
"format": "json",
"indent": 4,
}
}
@classmethod
def update_settings(cls, settings):
super().update_settings(settings)
settings.setdefault("FEEDS", {}).update(cls.custom_feed)
.. method:: start_requests()
This method must return an iterable with the first Requests to crawl for
@ -157,15 +212,24 @@ scrapy.Spider
If you want to change the Requests used to start scraping a domain, this is
the method to override. For example, if you need to start by logging in using
a POST request, you could do::
a POST request, you could do:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def start_requests(self):
return [scrapy.FormRequest("http://www.example.com/login",
formdata={'user': 'john', 'pass': 'secret'},
callback=self.logged_in)]
return [
scrapy.FormRequest(
"http://www.example.com/login",
formdata={"user": "john", "pass": "secret"},
callback=self.logged_in,
)
]
def logged_in(self, response):
# here you would extract links to follow and return Requests for
@ -200,63 +264,72 @@ scrapy.Spider
Called when the spider closes. This method provides a shortcut to
signals.connect() for the :signal:`spider_closed` signal.
Let's see an example::
Let's see an example:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
'http://www.example.com/3.html',
"http://www.example.com/1.html",
"http://www.example.com/2.html",
"http://www.example.com/3.html",
]
def parse(self, response):
self.logger.info('A response from %s just arrived!', response.url)
self.logger.info("A response from %s just arrived!", response.url)
Return multiple Requests and items from a single callback::
Return multiple Requests and items from a single callback:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
'http://www.example.com/3.html',
"http://www.example.com/1.html",
"http://www.example.com/2.html",
"http://www.example.com/3.html",
]
def parse(self, response):
for h3 in response.xpath('//h3').getall():
for h3 in response.xpath("//h3").getall():
yield {"title": h3}
for href in response.xpath('//a/@href').getall():
for href in response.xpath("//a/@href").getall():
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :class:`~scrapy.Item` objects::
to give data more structure you can use :class:`~scrapy.Item` objects:
.. skip: next
.. code-block:: python
import scrapy
from myproject.items import MyItem
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
def start_requests(self):
yield scrapy.Request('http://www.example.com/1.html', self.parse)
yield scrapy.Request('http://www.example.com/2.html', self.parse)
yield scrapy.Request('http://www.example.com/3.html', self.parse)
yield scrapy.Request("http://www.example.com/1.html", self.parse)
yield scrapy.Request("http://www.example.com/2.html", self.parse)
yield scrapy.Request("http://www.example.com/3.html", self.parse)
def parse(self, response):
for h3 in response.xpath('//h3').getall():
for h3 in response.xpath("//h3").getall():
yield MyItem(title=h3)
for href in response.xpath('//a/@href').getall():
for href in response.xpath("//a/@href").getall():
yield scrapy.Request(response.urljoin(href), self.parse)
.. _spiderargs:
@ -274,34 +347,43 @@ Spider arguments are passed through the :command:`crawl` command using the
scrapy crawl myspider -a category=electronics
Spiders can access arguments in their `__init__` methods::
Spiders can access arguments in their `__init__` methods:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = [f'http://www.example.com/categories/{category}']
self.start_urls = [f"http://www.example.com/categories/{category}"]
# ...
The default `__init__` method will take any spider arguments
and copy them to the spider as attributes.
The above example can also be written as follows::
The above example can also be written as follows:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def start_requests(self):
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
yield scrapy.Request(f"http://www.example.com/categories/{self.category}")
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
.. skip: next
.. code-block:: python
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
@ -337,10 +419,13 @@ common scraping cases, like following all links on a site based on certain
rules, crawling from `Sitemaps`_, or parsing an XML/CSV feed.
For the examples used in the following spiders, we'll assume you have a project
with a ``TestItem`` declared in a ``myproject.items`` module::
with a ``TestItem`` declared in a ``myproject.items`` module:
.. code-block:: python
import scrapy
class TestItem(scrapy.Item):
id = scrapy.Field()
name = scrapy.Field()
@ -436,38 +521,46 @@ Crawling rules
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
Let's now take a look at an example CrawlSpider with rules::
Let's now take a look at an example CrawlSpider with rules:
.. code-block:: python
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
class MySpider(CrawlSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com"]
rules = (
# Extract links matching 'category.php' (but not matching 'subsection.php')
# and follow links from them (since no callback means follow=True by default).
Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
Rule(LinkExtractor(allow=(r"category\.php",), deny=(r"subsection\.php",))),
# Extract links matching 'item.php' and parse them with the spider's method parse_item
Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
Rule(LinkExtractor(allow=(r"item\.php",)), callback="parse_item"),
)
def parse_item(self, response):
self.logger.info('Hi, this is an item page! %s', response.url)
self.logger.info("Hi, this is an item page! %s", response.url)
item = scrapy.Item()
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)")
item["name"] = response.xpath('//td[@id="item_name"]/text()').get()
item["description"] = response.xpath(
'//td[@id="item_description"]/text()'
).get()
item["link_text"] = response.meta["link_text"]
url = response.xpath('//td[@id="additional_data"]/@href').get()
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
return response.follow(
url, self.parse_additional_page, cb_kwargs=dict(item=item)
)
def parse_additional_page(self, response, item):
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
item["additional_data"] = response.xpath(
'//p[@id="additional_data"]/text()'
).get()
return item
@ -568,25 +661,31 @@ XMLFeedSpider
XMLFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
These spiders are pretty easy to use, let's have a look at one example::
These spiders are pretty easy to use, let's have a look at one example:
.. skip: next
.. code-block:: python
from scrapy.spiders import XMLFeedSpider
from myproject.items import TestItem
class MySpider(XMLFeedSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.xml']
iterator = 'iternodes' # This is actually unnecessary, since it's the default value
itertag = 'item'
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com/feed.xml"]
iterator = "iternodes" # This is actually unnecessary, since it's the default value
itertag = "item"
def parse_node(self, response, node):
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
self.logger.info(
"Hi, this is a <%s> node!: %s", self.itertag, "".join(node.getall())
)
item = TestItem()
item['id'] = node.xpath('@id').get()
item['name'] = node.xpath('name').get()
item['description'] = node.xpath('description').get()
item["id"] = node.xpath("@id").get()
item["name"] = node.xpath("name").get()
item["description"] = node.xpath("description").get()
return item
Basically what we did up there was to create a spider that downloads a feed from
@ -627,26 +726,30 @@ CSVFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
Let's see an example similar to the previous one, but using a
:class:`CSVFeedSpider`::
:class:`CSVFeedSpider`:
.. skip: next
.. code-block:: python
from scrapy.spiders import CSVFeedSpider
from myproject.items import TestItem
class MySpider(CSVFeedSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.csv']
delimiter = ';'
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com/feed.csv"]
delimiter = ";"
quotechar = "'"
headers = ['id', 'name', 'description']
headers = ["id", "name", "description"]
def parse_row(self, response, row):
self.logger.info('Hi, this is a row!: %r', row)
self.logger.info("Hi, this is a row!: %r", row)
item = TestItem()
item['id'] = row['id']
item['name'] = row['name']
item['description'] = row['description']
item["id"] = row["id"]
item["name"] = row["name"]
item["description"] = row["description"]
return item
@ -728,19 +831,22 @@ SitemapSpider
<lastmod>2005-01-01</lastmod>
</url>
We can define a ``sitemap_filter`` function to filter ``entries`` by date::
We can define a ``sitemap_filter`` function to filter ``entries`` by date:
.. code-block:: python
from datetime import datetime
from scrapy.spiders import SitemapSpider
class FilteredSitemapSpider(SitemapSpider):
name = 'filtered_sitemap_spider'
allowed_domains = ['example.com']
sitemap_urls = ['http://example.com/sitemap.xml']
name = "filtered_sitemap_spider"
allowed_domains = ["example.com"]
sitemap_urls = ["http://example.com/sitemap.xml"]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year >= 2005:
yield entry
@ -765,60 +871,72 @@ SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
Simplest example: process all urls discovered through sitemaps using the
``parse`` callback::
``parse`` callback:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/sitemap.xml']
sitemap_urls = ["http://www.example.com/sitemap.xml"]
def parse(self, response):
pass # ... scrape item here ...
pass # ... scrape item here ...
Process some urls with certain callback and other urls with a different
callback::
callback:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/sitemap.xml']
sitemap_urls = ["http://www.example.com/sitemap.xml"]
sitemap_rules = [
('/product/', 'parse_product'),
('/category/', 'parse_category'),
("/product/", "parse_product"),
("/category/", "parse_category"),
]
def parse_product(self, response):
pass # ... scrape product ...
pass # ... scrape product ...
def parse_category(self, response):
pass # ... scrape category ...
pass # ... scrape category ...
Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps
whose url contains ``/sitemap_shop``::
whose url contains ``/sitemap_shop``:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/robots.txt']
sitemap_urls = ["http://www.example.com/robots.txt"]
sitemap_rules = [
('/shop/', 'parse_shop'),
("/shop/", "parse_shop"),
]
sitemap_follow = ['/sitemap_shops']
sitemap_follow = ["/sitemap_shops"]
def parse_shop(self, response):
pass # ... scrape shop here ...
pass # ... scrape shop here ...
Combine SitemapSpider with other sources of urls::
Combine SitemapSpider with other sources of urls:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/robots.txt']
sitemap_urls = ["http://www.example.com/robots.txt"]
sitemap_rules = [
('/shop/', 'parse_shop'),
("/shop/", "parse_shop"),
]
other_urls = ['http://www.example.com/about']
other_urls = ["http://www.example.com/about"]
def start_requests(self):
requests = list(super(MySpider, self).start_requests())
@ -826,10 +944,10 @@ Combine SitemapSpider with other sources of urls::
return requests
def parse_shop(self, response):
pass # ... scrape shop here ...
pass # ... scrape shop here ...
def parse_other(self, response):
pass # ... scrape other here ...
pass # ... scrape other here ...
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index

41
docs/topics/stats.rst
View File

@ -30,10 +30,11 @@ Common Stats Collector uses
===========================
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
attribute. Here is an example of an extension that access stats:
.. code-block:: python
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats
@ -41,31 +42,43 @@ attribute. Here is an example of an extension that access stats::
def from_crawler(cls, crawler):
return cls(crawler.stats)
Set stat value::
Set stat value:
stats.set_value('hostname', socket.gethostname())
.. code-block:: python
Increment stat value::
stats.set_value("hostname", socket.gethostname())
stats.inc_value('custom_count')
Increment stat value:
Set stat value only if greater than previous::
.. code-block:: python
stats.max_value('max_items_scraped', value)
stats.inc_value("custom_count")
Set stat value only if lower than previous::
Set stat value only if greater than previous:
stats.min_value('min_free_memory_percent', value)
.. code-block:: python
stats.max_value("max_items_scraped", value)
Set stat value only if lower than previous:
.. code-block:: python
stats.min_value("min_free_memory_percent", value)
Get stat value:
>>> stats.get_value('custom_count')
1
.. code-block:: pycon
>>> stats.get_value("custom_count")
1
Get all stats:
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
.. code-block:: pycon
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================

8
docs/utils/linkfix.py
View File

@ -18,20 +18,19 @@ from pathlib import Path
def main():
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(r"(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))")
# Read lines from the linkcheck output file
try:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
except IOError:
except OSError:
print("linkcheck output not found; please run linkcheck first.")
sys.exit(1)
@ -50,7 +49,6 @@ def main():
else:
# If this is a new file
if newfilename != _filename:
# Update the previous file
if _filename:
Path(_filename).write_text(_contents, encoding="utf-8")
@ -66,5 +64,5 @@ def main():
print("Not Understood: " + line)
if __name__ == '__main__':
if __name__ == "__main__":
main()

22
extras/qps-bench-server.py
View File

@ -1,13 +1,13 @@
#!/usr/bin/env python
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from time import time
from twisted.internet import reactor
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
class Root(Resource):
def __init__(self):
Resource.__init__(self)
self.concurrent = 0
@ -26,9 +26,9 @@ class Root(Resource):
delta = now - self.lasttime
# reset stats on high iter-request times caused by client restarts
if delta > 3: # seconds
if delta > 3: # seconds
self._reset_stats()
return ''
return ""
self.tail.appendleft(delta)
self.lasttime = now
@ -37,15 +37,17 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
print(
f"samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}"
)
if 'latency' in request.args:
latency = float(request.args['latency'][0])
if "latency" in request.args:
latency = float(request.args["latency"][0])
reactor.callLater(latency, self._finish, request)
return NOT_DONE_YET
self.concurrent -= 1
return ''
return ""
def _finish(self, request):
self.concurrent -= 1

18
extras/qpsclient.py
View File

@ -1,25 +1,25 @@
"""
A spider that generate light requests to meassure QPS throughput
A spider that generate light requests to measure QPS throughput
usage:
scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0
--set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
"""
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.spiders import Spider
class QPSSpider(Spider):
name = 'qps'
benchurl = 'http://localhost:8880/'
name = "qps"
benchurl = "http://localhost:8880/"
# Max concurrency is limited by global CONCURRENT_REQUESTS setting
max_concurrent_requests = 8
# Requests per second goal
qps = None # same as: 1 / download_delay
qps = None # same as: 1 / download_delay
download_delay = None
# time in seconds to delay server responses
latency = None
@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += f'?latency={self.latency}'
url += f"?latency={self.latency}"
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
urls = [url.replace("localhost", f"127.0.0.{x + 1}") for x in range(slots)]
else:
urls = [url]

5
pylintrc
View File

@ -12,6 +12,7 @@ disable=abstract-method,
bad-mcs-classmethod-argument,
bare-except,
broad-except,
broad-exception-raised,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
@ -28,6 +29,7 @@ disable=abstract-method,
fixme,
function-redefined,
global-statement,
implicit-str-concat,
import-error,
import-outside-toplevel,
import-self,
@ -45,12 +47,14 @@ disable=abstract-method,
method-hidden,
missing-docstring,
no-else-raise,
no-else-return,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-value-for-parameter,
not-callable,
pointless-exception-statement,
pointless-statement,
pointless-string-statement,
protected-access,
@ -86,6 +90,7 @@ disable=abstract-method,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-dict-literal,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,

2
pytest.ini
View File

@ -5,7 +5,6 @@ python_files=test_*.py __init__.py
python_classes=
addopts =
--assert=plain
--doctest-modules
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
@ -21,6 +20,7 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
requires_uvloop: marks tests as only enabled when uvloop is known to be working
filterwarnings =
ignore:Module scrapy.utils.reqser is deprecated
ignore:scrapy.downloadermiddlewares.decompression is deprecated

2
scrapy/VERSION
View File

@ -1 +1 @@
2.7.1
2.11.0

26
scrapy/__init__.py
View File

@ -9,32 +9,38 @@ import warnings
from twisted import version as _txv
# Declare top-level shortcuts
from scrapy.spiders import Spider
from scrapy.http import Request, FormRequest
from scrapy.http import FormRequest, Request
from scrapy.item import Field, Item
from scrapy.selector import Selector
from scrapy.item import Item, Field
from scrapy.spiders import Spider
__all__ = [
'__version__', 'version_info', 'twisted_version', 'Spider',
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
"__version__",
"version_info",
"twisted_version",
"Spider",
"Request",
"FormRequest",
"Selector",
"Item",
"Field",
]
# Scrapy and Twisted versions
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 7):
print(f"Scrapy {__version__} requires Python 3.7+")
if sys.version_info < (3, 8):
print(f"Scrapy {__version__} requires Python 3.8+")
sys.exit(1)
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
del pkgutil

2
scrapy/__main__.py
View File

@ -1,4 +1,4 @@
from scrapy.cmdline import execute
if __name__ == '__main__':
if __name__ == "__main__":
execute()

51
scrapy/addons.py Normal file
View File

@ -0,0 +1,51 @@
import logging
from typing import TYPE_CHECKING, Any, List
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class AddonManager:
"""This class facilitates loading and storing :ref:`topics-addons`."""
def __init__(self, crawler: "Crawler") -> None:
self.crawler: "Crawler" = crawler
self.addons: List[Any] = []
def load_settings(self, settings: Settings) -> None:
"""Load add-ons and configurations from a settings object and apply them.
This will load the add-on for every add-on path in the
``ADDONS`` setting and execute their ``update_settings`` methods.
:param settings: The :class:`~scrapy.settings.Settings` object from \
which to read the add-on configuration
:type settings: :class:`~scrapy.settings.Settings`
"""
for clspath in build_component_list(settings["ADDONS"]):
try:
addoncls = load_object(clspath)
addon = build_from_crawler(addoncls, self.crawler)
addon.update_settings(settings)
self.addons.append(addon)
except NotConfigured as e:
if e.args:
logger.warning(
"Disabled %(clspath)s: %(eargs)s",
{"clspath": clspath, "eargs": e.args[0]},
extra={"crawler": self.crawler},
)
logger.info(
"Enabled addons:\n%(addons)s",
{
"addons": self.addons,
},
extra={"crawler": self.crawler},
)

50
scrapy/cmdline.py
View File

@ -1,30 +1,30 @@
import sys
import os
import argparse
import cProfile
import inspect
import pkg_resources
import os
import sys
from importlib.metadata import entry_points
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == '-:':
if arg_string[:2] == "-:":
return None
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# TODO: add `name` attribute to commands and merge this function with
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
for obj in vars(module).values():
@ -41,14 +41,18 @@ def _get_commands_from_module(module, inproject):
d = {}
for cmd in _iter_command_classes(module):
if inproject or not cmd.requires_project:
cmdname = cmd.__module__.split('.')[-1]
cmdname = cmd.__module__.split(".")[-1]
d[cmdname] = cmd()
return d
def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
cmds = {}
for entry_point in pkg_resources.iter_entry_points(group):
if sys.version_info >= (3, 10):
eps = entry_points(group=group)
else:
eps = entry_points().get(group, ())
for entry_point in eps:
obj = entry_point.load()
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
@ -58,9 +62,9 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
def _get_commands_dict(settings, inproject):
cmds = _get_commands_from_module('scrapy.commands', inproject)
cmds = _get_commands_from_module("scrapy.commands", inproject)
cmds.update(_get_commands_from_entry_points(inproject))
cmds_module = settings['COMMANDS_MODULE']
cmds_module = settings["COMMANDS_MODULE"]
if cmds_module:
cmds.update(_get_commands_from_module(cmds_module, inproject))
return cmds
@ -69,7 +73,7 @@ def _get_commands_dict(settings, inproject):
def _pop_command_name(argv):
i = 0
for arg in argv[1:]:
if not arg.startswith('-'):
if not arg.startswith("-"):
del argv[i]
return arg
i += 1
@ -124,11 +128,11 @@ def execute(argv=None, settings=None):
settings = get_project_settings()
# set EDITOR from environment if available
try:
editor = os.environ['EDITOR']
editor = os.environ["EDITOR"]
except KeyError:
pass
else:
settings['EDITOR'] = editor
settings["EDITOR"] = editor
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
@ -141,11 +145,13 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
parser = ScrapyArgumentParser(
formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler="resolve",
description=cmd.long_desc(),
)
settings.setdict(cmd.default_settings, priority="command")
cmd.settings = settings
cmd.add_options(parser)
opts, args = parser.parse_known_args(args=argv[1:])
@ -168,12 +174,12 @@ def _run_command_profiled(cmd, args, opts):
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)
p.runctx("cmd.run(args, opts)", globals(), loc)
if opts.profile:
p.dump_stats(opts.profile)
if __name__ == '__main__':
if __name__ == "__main__":
try:
execute()
finally:

134
scrapy/commands/__init__.py
View File

@ -1,20 +1,19 @@
"""
Base class for Scrapy commands
"""
import os
import argparse
import os
from pathlib import Path
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
from twisted.python import failure
from scrapy.crawler import CrawlerProcess
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
class ScrapyCommand:
requires_project = False
crawler_process: Optional[CrawlerProcess] = None
@ -27,7 +26,7 @@ class ScrapyCommand:
self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
if hasattr(self, '_crawler'):
if hasattr(self, "_crawler"):
raise RuntimeError("crawler already set")
self._crawler = crawler
@ -61,46 +60,63 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
group = parser.add_argument_group(title='Global Options')
group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_argument("--nolog", action="store_true",
help="disable logging completely")
group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group = parser.add_argument_group(title="Global Options")
group.add_argument(
"--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
)
group.add_argument(
"-L",
"--loglevel",
metavar="LEVEL",
default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})",
)
group.add_argument(
"--nolog", action="store_true", help="disable logging completely"
)
group.add_argument(
"--profile",
metavar="FILE",
default=None,
help="write python cProfile stats to FILE",
)
group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE")
group.add_argument(
"-s",
"--set",
action="append",
default=[],
metavar="NAME=VALUE",
help="set/override setting (may be repeated)",
)
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
self.settings.setdict(arglist_to_dict(opts.set),
priority='cmdline')
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
if opts.logfile:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
self.settings.set("LOG_ENABLED", True, priority="cmdline")
self.settings.set("LOG_FILE", opts.logfile, priority="cmdline")
if opts.loglevel:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
self.settings.set("LOG_ENABLED", True, priority="cmdline")
self.settings.set("LOG_LEVEL", opts.loglevel, priority="cmdline")
if opts.nolog:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
self.settings.set("LOG_ENABLED", False, priority="cmdline")
if opts.pidfile:
Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8")
Path(opts.pidfile).write_text(
str(os.getpid()) + os.linesep, encoding="utf-8"
)
if opts.pdb:
failure.startDebugMode()
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
"""
Entry point for running commands
"""
@ -111,18 +127,39 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
parser.add_argument(
"-a",
dest="spargs",
action="append",
default=[],
metavar="NAME=VALUE",
help="set spider argument (may be repeated)",
)
parser.add_argument(
"-o",
"--output",
metavar="FILE",
action="append",
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)",
)
parser.add_argument(
"-O",
"--overwrite-output",
metavar="FILE",
action="append",
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
)
parser.add_argument(
"-t",
"--output-format",
metavar="FORMAT",
help="format to use for dumping items",
)
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -137,16 +174,21 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
self.settings.set("FEEDS", feeds, priority="cmdline")
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
super().__init__(
prog,
indent_increment=indent_increment,
max_help_position=max_help_position,
width=width,
)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
@ -157,11 +199,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
part_strings[0] = "Usage\n=====\n " + part_strings[0][len("usage: ") :]
headings = [
i for i in range(len(part_strings)) if part_strings[i].endswith(":\n")
]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
char = "-" if "Global Options" in part_strings[index] else "="
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
underline = "".join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

25
scrapy/commands/bench.py
View File

@ -1,6 +1,6 @@
import subprocess
import sys
import time
import subprocess
from urllib.parse import urlencode
import scrapy
@ -9,11 +9,10 @@ from scrapy.linkextractors import LinkExtractor
class Command(ScrapyCommand):
default_settings = {
'LOG_LEVEL': 'INFO',
'LOGSTATS_INTERVAL': 1,
'CLOSESPIDER_TIMEOUT': 10,
"LOG_LEVEL": "INFO",
"LOGSTATS_INTERVAL": 1,
"CLOSESPIDER_TIMEOUT": 10,
}
def short_desc(self):
@ -26,12 +25,11 @@ class Command(ScrapyCommand):
class _BenchServer:
def __enter__(self):
from scrapy.utils.test import get_testenv
pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver']
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE,
env=get_testenv())
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
self.proc.stdout.readline()
def __exit__(self, exc_type, exc_value, traceback):
@ -42,15 +40,16 @@ class _BenchServer:
class _BenchSpider(scrapy.Spider):
"""A spider that follows all links"""
name = 'follow'
name = "follow"
total = 10000
show = 20
baseurl = 'http://localhost:8998'
baseurl = "http://localhost:8998"
link_extractor = LinkExtractor()
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
qargs = {"total": self.total, "show": self.show}
url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

32
scrapy/commands/check.py
View File

@ -1,11 +1,12 @@
import time
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
from unittest import TextTestResult as _TextTestResult
from unittest import TextTestRunner
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object, set_environ
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):
@ -39,7 +40,7 @@ class TextTestResult(_TextTestResult):
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "[options] <spider>"
@ -49,14 +50,25 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
parser.add_argument(
"-l",
"--list",
dest="list",
action="store_true",
help="only list contracts, without checking them",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
default=False,
action="store_true",
help="print contract tests for all spiders",
)
def run(self, args, opts):
# load contracts
contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
conman = ContractsManager(load_object(c) for c in contracts)
runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
@ -66,7 +78,7 @@ class Command(ScrapyCommand):
spider_loader = self.crawler_process.spider_loader
with set_environ(SCRAPY_CHECK='true'):
with set_environ(SCRAPY_CHECK="true"):
for spidername in args or spider_loader.list():
spidercls = spider_loader.load(spidername)
spidercls.start_requests = lambda s: conman.from_spider(s, result)
@ -85,7 +97,7 @@ class Command(ScrapyCommand):
continue
print(spider)
for method in sorted(methods):
print(f' * {method}')
print(f" * {method}")
else:
start = time.time()
self.crawler_process.start()

12
scrapy/commands/crawl.py
View File

@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError
class Command(BaseRunSpiderCommand):
requires_project = True
def syntax(self):
@ -16,18 +15,23 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
raise UsageError(
"running 'scrapy crawl' with more than one spider is not supported"
)
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
if getattr(crawl_defer, "result", None) is not None and issubclass(
crawl_defer.result.type, Exception
):
self.exitcode = 1
else:
self.crawler_process.start()
if (
self.crawler_process.bootstrap_failed
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
or hasattr(self.crawler_process, "has_exception")
and self.crawler_process.has_exception
):
self.exitcode = 1

15
scrapy/commands/edit.py
View File

@ -1,14 +1,13 @@
import sys
import os
import sys
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "<spider>"
@ -17,8 +16,10 @@ class Command(ScrapyCommand):
return "Edit spider"
def long_desc(self):
return ("Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting")
return (
"Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting"
)
def _err(self, msg):
sys.stderr.write(msg + os.linesep)
@ -28,12 +29,12 @@ class Command(ScrapyCommand):
if len(args) != 1:
raise UsageError()
editor = self.settings['EDITOR']
editor = self.settings["EDITOR"]
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
sfile = sfile.replace(".pyc", ".py")
self.exitcode = os.system(f'{editor} "{sfile}"')

53
scrapy/commands/fetch.py
View File

@ -1,15 +1,18 @@
import sys
from argparse import Namespace
from typing import List, Type
from w3lib.url import is_url
from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
class Command(ScrapyCommand):
requires_project = False
def syntax(self):
@ -27,40 +30,54 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument(
"--headers",
dest="headers",
action="store_true",
help="print response HTTP headers instead of body",
)
parser.add_argument(
"--no-redirect",
dest="no_redirect",
action="store_true",
default=False,
help="do not handle HTTP 3xx status codes and print response as-is",
)
def _print_headers(self, headers, prefix):
for key, values in headers.items():
for value in values:
self._print_bytes(prefix + b' ' + key + b': ' + value)
self._print_bytes(prefix + b" " + key + b": " + value)
def _print_response(self, response, opts):
if opts.headers:
self._print_headers(response.request.headers, b'>')
print('>')
self._print_headers(response.headers, b'<')
self._print_headers(response.request.headers, b">")
print(">")
self._print_headers(response.headers, b"<")
else:
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
sys.stdout.buffer.write(bytes_ + b'\n')
sys.stdout.buffer.write(bytes_ + b"\n")
def run(self, args, opts):
def run(self, args: List[str], opts: Namespace) -> None:
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
request = Request(
args[0],
callback=self._print_response,
cb_kwargs={"opts": opts},
dont_filter=True,
)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:
request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
request.meta["handle_httpstatus_list"] = SequenceExclude(range(300, 400))
else:
request.meta['handle_httpstatus_all'] = True
request.meta["handle_httpstatus_all"] = True
spidercls = DefaultSpider
spidercls: Type[Spider] = DefaultSpider
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
if opts.spider:
spidercls = spider_loader.load(opts.spider)

116
scrapy/commands/genspider.py
View File

@ -1,16 +1,15 @@
import os
import shutil
import string
from pathlib import Path
from importlib import import_module
from pathlib import Path
from typing import Optional, cast
from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
def sanitize_module_name(module_name):
@ -18,7 +17,7 @@ def sanitize_module_name(module_name):
with underscores and prefixing it with a letter if it doesn't start
with one
"""
module_name = module_name.replace('-', '_').replace('.', '_')
module_name = module_name.replace("-", "_").replace(".", "_")
if module_name[0] not in string.ascii_letters:
module_name = "a" + module_name
return module_name
@ -27,15 +26,22 @@ def sanitize_module_name(module_name):
def extract_domain(url):
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == '' and o.netloc == '':
if o.scheme == "" and o.netloc == "":
o = urlparse("//" + url.lstrip("/"))
return o.netloc
class Command(ScrapyCommand):
def verify_url_scheme(url):
"""Check url for scheme and insert https if none found."""
parsed = urlparse(url)
if parsed.scheme == "" and parsed.netloc == "":
parsed = urlparse("//" + url)._replace(scheme="https")
return parsed.geturl()
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "[options] <name> <domain>"
@ -45,16 +51,40 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
parser.add_argument(
"-l",
"--list",
dest="list",
action="store_true",
help="List available templates",
)
parser.add_argument(
"-e",
"--edit",
dest="edit",
action="store_true",
help="Edit spider after creating it",
)
parser.add_argument(
"-d",
"--dump",
dest="dump",
metavar="TEMPLATE",
help="Dump template to standard output",
)
parser.add_argument(
"-t",
"--template",
dest="template",
default="basic",
help="Uses a custom template.",
)
parser.add_argument(
"--force",
dest="force",
action="store_true",
help="If the spider already exists, overwrite it with the template",
)
def run(self, args, opts):
if opts.list:
@ -69,10 +99,10 @@ class Command(ScrapyCommand):
raise UsageError()
name, url = args[0:2]
domain = extract_domain(url)
url = verify_url_scheme(url)
module = sanitize_module_name(name)
if self.settings.get('BOT_NAME') == module:
if self.settings.get("BOT_NAME") == module:
print("Cannot create a spider with the same name as your project")
return
@ -81,23 +111,25 @@ class Command(ScrapyCommand):
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
def _genspider(self, module, name, url, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
domain = extract_domain(url)
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': f'{capitalized_module}Spider'
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
"url": url,
"domain": domain,
"classname": f"{capitalized_module}Spider",
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
@ -105,13 +137,15 @@ class Command(ScrapyCommand):
spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
print(
f"Created spider {name!r} using template {template_name!r} ",
end=("" if spiders_module else "\n"),
)
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template: str) -> Optional[Path]:
template_file = Path(self.templates_dir, f'{template}.tmpl')
template_file = Path(self.templates_dir, f"{template}.tmpl")
if template_file.exists():
return template_file
print(f"Unable to find template: {template}\n")
@ -121,11 +155,11 @@ class Command(ScrapyCommand):
def _list_templates(self):
print("Available templates:")
for file in sorted(Path(self.templates_dir).iterdir()):
if file.suffix == '.tmpl':
if file.suffix == ".tmpl":
print(f" {file.stem}")
def _spider_exists(self, name: str) -> bool:
if not self.settings.get('NEWSPIDER_MODULE'):
if not self.settings.get("NEWSPIDER_MODULE"):
# if run as a standalone command and file with same filename already exists
path = Path(name + ".py")
if path.exists():
@ -148,7 +182,7 @@ class Command(ScrapyCommand):
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(cast(str, spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
@ -160,7 +194,9 @@ class Command(ScrapyCommand):
@property
def templates_dir(self) -> str:
return str(Path(
self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'spiders'
))
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
"spiders",
)
)

3
scrapy/commands/list.py
View File

@ -2,9 +2,8 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def short_desc(self):
return "List available spiders"

187
scrapy/commands/parse.py
View File

@ -1,16 +1,21 @@
import inspect
import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import maybeDeferred
from w3lib.url import is_url
from twisted.internet.defer import maybeDeferred
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
logger = logging.getLogger(__name__)
@ -32,28 +37,72 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %(default)s]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
parser.add_argument(
"--spider",
dest="spider",
default=None,
help="use this spider without looking for one",
)
parser.add_argument(
"--pipelines", action="store_true", help="process items through pipelines"
)
parser.add_argument(
"--nolinks",
dest="nolinks",
action="store_true",
help="don't show links to follow (extracted requests)",
)
parser.add_argument(
"--noitems",
dest="noitems",
action="store_true",
help="don't show scraped items",
)
parser.add_argument(
"--nocolour",
dest="nocolour",
action="store_true",
help="avoid using pygments to colorize the output",
)
parser.add_argument(
"-r",
"--rules",
dest="rules",
action="store_true",
help="use CrawlSpider rules to discover the callback",
)
parser.add_argument(
"-c",
"--callback",
dest="callback",
help="use this callback for parsing, instead looking for a callback",
)
parser.add_argument(
"-m",
"--meta",
dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string",
)
parser.add_argument(
"--cbkwargs",
dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string",
)
parser.add_argument(
"-d",
"--depth",
dest="depth",
type=int,
default=1,
help="maximum depth for parsing requests [default: %(default)s]",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
action="store_true",
help="print each depth level one by one",
)
@property
def max_level(self):
@ -64,6 +113,25 @@ class Command(BaseRunSpiderCommand):
max_requests = max(self.requests)
return max(max_items, max_requests)
def handle_exception(self, _failure):
logger.error(
"An error is caught while iterating the async iterable",
exc_info=failure_to_exc_info(_failure),
)
def iterate_spider_output(self, result):
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))
)
d.addCallback(self.iterate_spider_output)
return d
if inspect.iscoroutine(result):
d = deferred_from_coro(result)
d.addCallback(self.iterate_spider_output)
return d
return arg_to_iter(deferred_from_coro(result))
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
self.items[lvl] = old_items + new_items
@ -98,13 +166,13 @@ class Command(BaseRunSpiderCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print(f'\n>>> DEPTH LEVEL: {level} <<<')
print(f"\n>>> DEPTH LEVEL: {level} <<<")
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
print(f"\n>>> STATUS DEPTH LEVEL {self.max_level} <<<")
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:
@ -121,18 +189,20 @@ class Command(BaseRunSpiderCommand):
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(self, spider, response):
if getattr(spider, 'rules', None):
if getattr(spider, "rules", None):
for rule in spider.rules:
if rule.link_extractor.matches(response.url):
return rule.callback or "parse"
else:
logger.error('No CrawlSpider rules found in spider %(spider)r, '
'please specify a callback to use for parsing',
{'spider': spider.name})
logger.error(
"No CrawlSpider rules found in spider %(spider)r, "
"please specify a callback to use for parsing",
{"spider": spider.name},
)
def set_spidercls(self, url, opts):
spider_loader = self.crawler_process.spider_loader
@ -140,15 +210,17 @@ class Command(BaseRunSpiderCommand):
try:
self.spidercls = spider_loader.load(opts.spider)
except KeyError:
logger.error('Unable to find spider: %(spider)s',
{'spider': opts.spider})
logger.error(
"Unable to find spider: %(spider)s", {"spider": opts.spider}
)
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
logger.error("Unable to find spider for: %(url)s", {"url": url})
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
if self.spidercls:
self.spidercls.start_requests = _start_requests
@ -158,8 +230,7 @@ class Command(BaseRunSpiderCommand):
self.crawler_process.start()
if not self.first_response:
logger.error('No response downloaded for: %(url)s',
{'url': url})
logger.error("No response downloaded for: %(url)s", {"url": url})
def scraped_data(self, args):
items, requests, opts, depth, spider, callback = args
@ -173,8 +244,8 @@ class Command(BaseRunSpiderCommand):
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.meta["_depth"] = depth + 1
req.meta["_callback"] = req.callback
req.callback = callback
scraped_data += requests
@ -187,7 +258,7 @@ class Command(BaseRunSpiderCommand):
self.first_response = response
# determine real callback
cb = response.meta['_callback']
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
@ -195,23 +266,27 @@ class Command(BaseRunSpiderCommand):
cb = self.get_callback_from_rules(spider, response)
if not cb:
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
{'url': response.url, 'spider': spider.name})
logger.error(
"Cannot find a rule that matches %(url)r in spider: %(spider)s",
{"url": response.url, "spider": spider.name},
)
return
else:
cb = 'parse'
cb = "parse"
if not callable(cb):
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
{'callback': cb, 'spider': spider.name})
logger.error(
"Cannot find callback %(callback)r in spider: %(spider)s",
{"callback": cb, "spider": spider.name},
)
return
# parse items and requests
depth = response.meta['_depth']
depth = response.meta["_depth"]
d = self.run_callback(response, cb, cb_kwargs)
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
@ -226,8 +301,8 @@ class Command(BaseRunSpiderCommand):
if opts.cbkwargs:
request.cb_kwargs.update(opts.cbkwargs)
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.meta["_depth"] = 1
request.meta["_callback"] = request.callback
request.callback = callback
return request
@ -242,16 +317,22 @@ class Command(BaseRunSpiderCommand):
try:
opts.meta = json.loads(opts.meta)
except ValueError:
raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
"Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
raise UsageError(
"Invalid -m/--meta value, pass a valid json string to -m or --meta. "
'Example: --meta=\'{"foo" : "bar"}\'',
print_help=False,
)
def process_request_cb_kwargs(self, opts):
if opts.cbkwargs:
try:
opts.cbkwargs = json.loads(opts.cbkwargs)
except ValueError:
raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
"Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
raise UsageError(
"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
'Example: --cbkwargs=\'{"foo" : "bar"}\'',
print_help=False,
)
def run(self, args, opts):
# parse arguments

11
scrapy/commands/runspider.py
View File

@ -1,18 +1,18 @@
import sys
from importlib import import_module
from os import PathLike
from pathlib import Path
from importlib import import_module
from types import ModuleType
from typing import Union
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in ('.py', '.pyw'):
if abspath.suffix not in (".py", ".pyw"):
raise ValueError(f"Not a Python source file: {abspath}")
dirname = str(abspath.parent)
sys.path = [dirname] + sys.path
@ -24,9 +24,8 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[options] <spider_file>"

41
scrapy/commands/settings.py
View File

@ -5,10 +5,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[options]"
@ -18,16 +16,33 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
parser.add_argument(
"--get", dest="get", metavar="SETTING", help="print raw setting value"
)
parser.add_argument(
"--getbool",
dest="getbool",
metavar="SETTING",
help="print setting value, interpreted as a boolean",
)
parser.add_argument(
"--getint",
dest="getint",
metavar="SETTING",
help="print setting value, interpreted as an integer",
)
parser.add_argument(
"--getfloat",
dest="getfloat",
metavar="SETTING",
help="print setting value, interpreted as a float",
)
parser.add_argument(
"--getlist",
dest="getlist",
metavar="SETTING",
help="print setting value, interpreted as a list",
)
def run(self, args, opts):
settings = self.crawler_process.settings

54
scrapy/commands/shell.py
View File

@ -3,22 +3,24 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
from argparse import Namespace
from threading import Thread
from typing import List, Type
from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.shell import Shell
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
class Command(ScrapyCommand):
requires_project = False
default_settings = {
'KEEP_ALIVE': True,
'LOGSTATS_INTERVAL': 0,
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
"KEEP_ALIVE": True,
"LOGSTATS_INTERVAL": 0,
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
}
def syntax(self):
@ -28,17 +30,26 @@ class Command(ScrapyCommand):
return "Interactive scraping console"
def long_desc(self):
return ("Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file.")
return (
"Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file."
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_argument("--spider", dest="spider",
help="use this spider")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument(
"-c",
dest="code",
help="evaluate the code in the shell, print the result and exit",
)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument(
"--no-redirect",
dest="no_redirect",
action="store_true",
default=False,
help="do not handle HTTP 3xx status codes and print response as-is",
)
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
@ -46,24 +57,27 @@ class Command(ScrapyCommand):
"""
pass
def run(self, args, opts):
def run(self, args: List[str], opts: Namespace) -> None:
url = args[0] if args else None
if url:
# first argument may be a local file
url = guess_scheme(url)
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
spidercls = DefaultSpider
spidercls: Type[Spider] = DefaultSpider
if opts.spider:
spidercls = spider_loader.load(opts.spider)
elif url:
spidercls = spidercls_for_request(spider_loader, Request(url),
spidercls, log_multiple=True)
spidercls = spidercls_for_request(
spider_loader, Request(url), spidercls, log_multiple=True
)
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
crawler = self.crawler_process._create_crawler(spidercls)
crawler._apply_settings()
# The Shell class needs a persistent engine in the crawler
crawler.engine = crawler._create_engine()
crawler.engine.start()
@ -74,7 +88,9 @@ class Command(ScrapyCommand):
shell.start(url=url, redirect=not opts.no_redirect)
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
t = Thread(
target=self.crawler_process.start,
kwargs={"stop_after_crawl": False, "install_signal_handlers": False},
)
t.daemon = True
t.start()

69
scrapy/commands/startproject.py
View File

@ -1,26 +1,25 @@
import re
import os
import re
import string
from importlib.util import find_spec
from pathlib import Path
from shutil import ignore_patterns, move, copy2, copystat
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
TEMPLATES_TO_RENDER = (
('scrapy.cfg',),
('${project_name}', 'settings.py.tmpl'),
('${project_name}', 'items.py.tmpl'),
('${project_name}', 'pipelines.py.tmpl'),
('${project_name}', 'middlewares.py.tmpl'),
("scrapy.cfg",),
("${project_name}", "settings.py.tmpl"),
("${project_name}", "items.py.tmpl"),
("${project_name}", "pipelines.py.tmpl"),
("${project_name}", "middlewares.py.tmpl"),
)
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
def _make_writable(path):
@ -29,10 +28,8 @@ def _make_writable(path):
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "<project_name> [project_dir]"
@ -45,11 +42,13 @@ class Command(ScrapyCommand):
spec = find_spec(module_name)
return spec is not None and spec.loader is not None
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
if not re.search(r"^[_a-zA-Z]\w*$", project_name):
print(
"Error: Project names must begin with a letter and contain"
" only\nletters, numbers and underscores"
)
elif _module_exists(project_name):
print(f'Error: Module {project_name!r} already exists')
print(f"Error: Module {project_name!r} already exists")
else:
return True
return False
@ -96,9 +95,9 @@ class Command(ScrapyCommand):
else:
project_dir = Path(args[0])
if (project_dir / 'scrapy.cfg').exists():
if (project_dir / "scrapy.cfg").exists():
self.exitcode = 1
print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')
print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}")
return
if not self._is_valid_name(project_name):
@ -106,12 +105,24 @@ class Command(ScrapyCommand):
return
self._copytree(Path(self.templates_dir), project_dir.resolve())
move(project_dir / 'module', project_dir / project_name)
move(project_dir / "module", project_dir / project_name)
for paths in TEMPLATES_TO_RENDER:
tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
tplfile = Path(
project_dir,
*(
string.Template(s).substitute(project_name=project_name)
for s in paths
),
)
render_templatefile(
tplfile,
project_name=project_name,
ProjectName=string_camelcase(project_name),
)
print(
f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:"
)
print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:")
print(f" cd {project_dir}")
@ -119,7 +130,9 @@ class Command(ScrapyCommand):
@property
def templates_dir(self) -> str:
return str(Path(
self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'project'
))
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
"project",
)
)

13
scrapy/commands/version.py
View File

@ -4,9 +4,7 @@ from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[-v]"
@ -16,8 +14,13 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store_true",
help="also display twisted/python/platform info (useful for bug reports)",
)
def run(self, args, opts):
if opts.verbose:

8
scrapy/commands/view.py
View File

@ -1,19 +1,21 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
class Command(fetch.Command):
def short_desc(self):
return "Open URL in browser, as seen by Scrapy"
def long_desc(self):
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
return (
"Fetch a URL using the Scrapy downloader and show its contents in a browser"
)
def add_options(self, parser):
super().add_options(parser)
parser.add_argument('--headers', help=argparse.SUPPRESS)
parser.add_argument("--headers", help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)

54
scrapy/contracts/__init__.py
View File

@ -2,7 +2,8 @@ import re
import sys
from functools import wraps
from inspect import getmembers
from typing import Dict
from types import CoroutineType
from typing import AsyncGenerator, Dict, Optional, Type
from unittest import TestCase
from scrapy.http import Request
@ -11,16 +12,17 @@ from scrapy.utils.spider import iterate_spider_output
class Contract:
""" Abstract class for contracts """
request_cls = None
"""Abstract class for contracts"""
request_cls: Optional[Type[Request]] = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
if hasattr(self, "pre_process"):
cb = request.callback
@wraps(cb)
@ -36,19 +38,27 @@ class Contract:
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
return list( # pylint: disable=return-in-finally
iterate_spider_output(cb_result)
)
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
if hasattr(self, "post_process"):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
output = list(iterate_spider_output(cb_result))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@ -60,7 +70,7 @@ class Contract:
else:
results.addSuccess(self.testcase_post)
finally:
return output
return output # pylint: disable=return-in-finally
request.callback = wrapper
@ -88,12 +98,12 @@ class ContractsManager:
def extract_contracts(self, method):
contracts = []
for line in method.__doc__.split('\n'):
for line in method.__doc__.split("\n"):
line = line.strip()
if line.startswith('@'):
name, args = re.match(r'@(\w+)\s*(.*)', line).groups()
args = re.split(r'\s+', args)
if line.startswith("@"):
name, args = re.match(r"@(\w+)\s*(.*)", line).groups()
args = re.split(r"\s+", args)
contracts.append(self.contracts[name](method, *args))
@ -106,7 +116,7 @@ class ContractsManager:
try:
requests.append(self.from_method(bound_method, results))
except Exception:
case = _create_testcase(bound_method, 'contract')
case = _create_testcase(bound_method, "contract")
results.addError(case, sys.exc_info())
return requests
@ -124,13 +134,13 @@ class ContractsManager:
# Don't filter requests to allow
# testing different callbacks on the same URL.
kwargs['dont_filter'] = True
kwargs['callback'] = method
kwargs["dont_filter"] = True
kwargs["callback"] = method
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
args.remove('self')
args.remove("self")
# check if all positional arguments are defined in kwargs
if set(args).issubset(set(kwargs)):
@ -146,7 +156,7 @@ class ContractsManager:
return request
def _clean_req(self, request, method, results):
""" stop the request from returning objects and records any errors """
"""stop the request from returning objects and records any errors"""
cb = request.callback
@ -156,11 +166,11 @@ class ContractsManager:
output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
except Exception:
case = _create_testcase(method, 'callback')
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())
def eb_wrapper(failure):
case = _create_testcase(method, 'errback')
case = _create_testcase(method, "errback")
exc_info = failure.type, failure.value, failure.getTracebackObject()
results.addError(case, exc_info)
@ -175,6 +185,6 @@ def _create_testcase(method, desc):
def __str__(_self):
return f"[{spider}] {method.__name__} ({desc})"
name = f'{spider}_{method.__name__}'
name = f"{spider}_{method.__name__}"
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

62
scrapy/contracts/default.py
View File

@ -1,6 +1,6 @@
import json
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
@ -9,50 +9,50 @@ from scrapy.http import Request
# contracts
class UrlContract(Contract):
""" Contract to set the url of the request (mandatory)
@url http://scrapy.org
"""Contract to set the url of the request (mandatory)
@url http://scrapy.org
"""
name = 'url'
name = "url"
def adjust_request_args(self, args):
args['url'] = self.args[0]
args["url"] = self.args[0]
return args
class CallbackKeywordArgumentsContract(Contract):
""" Contract to set the keyword arguments for the request.
The value should be a JSON-encoded dictionary, e.g.:
"""Contract to set the keyword arguments for the request.
The value should be a JSON-encoded dictionary, e.g.:
@cb_kwargs {"arg1": "some value"}
@cb_kwargs {"arg1": "some value"}
"""
name = 'cb_kwargs'
name = "cb_kwargs"
def adjust_request_args(self, args):
args['cb_kwargs'] = json.loads(' '.join(self.args))
args["cb_kwargs"] = json.loads(" ".join(self.args))
return args
class ReturnsContract(Contract):
""" Contract to check the output of a callback
"""Contract to check the output of a callback
general form:
@returns request(s)/item(s) [min=1 [max]]
general form:
@returns request(s)/item(s) [min=1 [max]]
e.g.:
@returns request
@returns request 2
@returns request 2 10
@returns request 0 10
e.g.:
@returns request
@returns request 2
@returns request 2 10
@returns request 0 10
"""
name = 'returns'
name = "returns"
object_type_verifiers = {
'request': lambda x: isinstance(x, Request),
'requests': lambda x: isinstance(x, Request),
'item': is_item,
'items': is_item,
"request": lambda x: isinstance(x, Request),
"requests": lambda x: isinstance(x, Request),
"item": is_item,
"items": is_item,
}
def __init__(self, *args, **kwargs):
@ -73,7 +73,7 @@ class ReturnsContract(Contract):
try:
self.max_bound = int(self.args[2])
except IndexError:
self.max_bound = float('inf')
self.max_bound = float("inf")
def post_process(self, output):
occurrences = 0
@ -81,23 +81,25 @@ class ReturnsContract(Contract):
if self.obj_type_verifier(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
assertion = self.min_bound <= occurrences <= self.max_bound
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = f'{self.min_bound}..{self.max_bound}'
expected = f"{self.min_bound}..{self.max_bound}"
raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
raise ContractFail(
f"Returned {occurrences} {self.obj_name}, expected {expected}"
)
class ScrapesContract(Contract):
""" Contract to check presence of fields in scraped items
@scrapes page_name page_body
"""Contract to check presence of fields in scraped items
@scrapes page_name page_body
"""
name = 'scrapes'
name = "scrapes"
def post_process(self, output):
for x in output:

173
scrapy/core/downloader/__init__.py
View File

@ -1,51 +1,61 @@
import random
from time import time
from datetime import datetime
from collections import deque
from datetime import datetime
from time import time
from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast
from twisted.internet import defer, task
from twisted.internet import task
from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.settings import BaseSettings
from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class Slot:
"""Downloader slot"""
def __init__(self, concurrency, delay, randomize_delay):
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
def __init__(self, concurrency: int, delay: float, randomize_delay: bool):
self.concurrency: int = concurrency
self.delay: float = delay
self.randomize_delay: bool = randomize_delay
self.active = set()
self.queue = deque()
self.transferring = set()
self.lastseen = 0
self.active: Set[Request] = set()
self.queue: Deque[Tuple[Request, Deferred]] = deque()
self.transferring: Set[Request] = set()
self.lastseen: float = 0
self.latercall = None
def free_transfer_slots(self):
def free_transfer_slots(self) -> int:
return self.concurrency - len(self.transferring)
def download_delay(self):
def download_delay(self) -> float:
if self.randomize_delay:
return random.uniform(0.5 * self.delay, 1.5 * self.delay)
return self.delay
def close(self):
def close(self) -> None:
if self.latercall and self.latercall.active():
self.latercall.cancel()
def __repr__(self):
def __repr__(self) -> str:
cls_name = self.__class__.__name__
return (f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})")
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})"
)
def __str__(self):
def __str__(self) -> str:
return (
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
@ -55,37 +65,45 @@ class Slot:
)
def _get_concurrency_delay(concurrency, spider, settings):
delay = settings.getfloat('DOWNLOAD_DELAY')
if hasattr(spider, 'download_delay'):
def _get_concurrency_delay(
concurrency: int, spider: Spider, settings: BaseSettings
) -> Tuple[int, float]:
delay: float = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
delay = spider.download_delay
if hasattr(spider, 'max_concurrent_requests'):
if hasattr(spider, "max_concurrent_requests"):
concurrency = spider.max_concurrent_requests
return concurrency, delay
class Downloader:
DOWNLOAD_SLOT = "download_slot"
DOWNLOAD_SLOT = 'download_slot'
def __init__(self, crawler):
self.settings = crawler.settings
self.signals = crawler.signals
self.slots = {}
self.active = set()
self.handlers = DownloadHandlers(crawler)
self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
self._slot_gc_loop = task.LoopingCall(self._slot_gc)
def __init__(self, crawler: "Crawler"):
self.settings: BaseSettings = crawler.settings
self.signals: SignalManager = crawler.signals
self.slots: Dict[str, Slot] = {}
self.active: Set[Request] = set()
self.handlers: DownloadHandlers = DownloadHandlers(crawler)
self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS")
self.domain_concurrency: int = self.settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
)
self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc)
self._slot_gc_loop.start(60)
self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict(
"DOWNLOAD_SLOTS", {}
)
def fetch(self, request, spider):
def _deactivate(response):
def fetch(self, request: Request, spider: Spider) -> Deferred:
def _deactivate(response: Response) -> Response:
self.active.remove(request)
return response
@ -93,47 +111,57 @@ class Downloader:
dfd = self.middleware.download(self._enqueue_request, request, spider)
return dfd.addBoth(_deactivate)
def needs_backout(self):
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
def _get_slot(self, request, spider):
def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
key = self._get_slot_key(request, spider)
if key not in self.slots:
conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
slot_settings = self.per_slot_settings.get(key, {})
conc = (
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
)
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
self.slots[key] = Slot(conc, delay, self.randomize_delay)
conc, delay = (
slot_settings.get("concurrency", conc),
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot
return key, self.slots[key]
def _get_slot_key(self, request, spider):
def _get_slot_key(self, request: Request, spider: Spider) -> str:
if self.DOWNLOAD_SLOT in request.meta:
return request.meta[self.DOWNLOAD_SLOT]
return cast(str, request.meta[self.DOWNLOAD_SLOT])
key = urlparse_cached(request).hostname or ''
key = urlparse_cached(request).hostname or ""
if self.ip_concurrency:
key = dnscache.get(key, key)
return key
def _enqueue_request(self, request, spider):
def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
key, slot = self._get_slot(request, spider)
request.meta[self.DOWNLOAD_SLOT] = key
def _deactivate(response):
def _deactivate(response: Response) -> Response:
slot.active.remove(request)
return response
slot.active.add(request)
self.signals.send_catch_log(signal=signals.request_reached_downloader,
request=request,
spider=spider)
deferred = defer.Deferred().addBoth(_deactivate)
self.signals.send_catch_log(
signal=signals.request_reached_downloader, request=request, spider=spider
)
deferred: Deferred = Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred))
self._process_queue(spider, slot)
return deferred
def _process_queue(self, spider, slot):
def _process_queue(self, spider: Spider, slot: Slot) -> None:
from twisted.internet import reactor
if slot.latercall and slot.latercall.active():
return
@ -143,7 +171,9 @@ class Downloader:
if delay:
penalty = delay - now + slot.lastseen
if penalty > 0:
slot.latercall = reactor.callLater(penalty, self._process_queue, spider, slot)
slot.latercall = reactor.callLater(
penalty, self._process_queue, spider, slot
)
return
# Process enqueued requests if there are free slots to transfer for this slot
@ -157,7 +187,7 @@ class Downloader:
self._process_queue(spider, slot)
break
def _download(self, slot, request, spider):
def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred:
# The order is very important for the following deferreds. Do not change!
# 1. Create the download deferred
@ -165,12 +195,15 @@ class Downloader:
# 2. Notify response_downloaded listeners about the recent download
# before querying queue for next request
def _downloaded(response):
self.signals.send_catch_log(signal=signals.response_downloaded,
response=response,
request=request,
spider=spider)
def _downloaded(response: Response) -> Response:
self.signals.send_catch_log(
signal=signals.response_downloaded,
response=response,
request=request,
spider=spider,
)
return response
dfd.addCallback(_downloaded)
# 3. After response arrives, remove the request from transferring
@ -179,22 +212,22 @@ class Downloader:
# middleware itself)
slot.transferring.add(request)
def finish_transferring(_):
def finish_transferring(_: Any) -> Any:
slot.transferring.remove(request)
self._process_queue(spider, slot)
self.signals.send_catch_log(signal=signals.request_left_downloader,
request=request,
spider=spider)
self.signals.send_catch_log(
signal=signals.request_left_downloader, request=request, spider=spider
)
return _
return dfd.addBoth(finish_transferring)
def close(self):
def close(self) -> None:
self._slot_gc_loop.stop()
for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
def _slot_gc(self, age: float = 60) -> None:
mintime = time() - age
for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime:

109
scrapy/core/downloader/contextfactory.py
View File

@ -1,15 +1,29 @@
import warnings
from typing import TYPE_CHECKING, Any, List, Optional
from OpenSSL import SSL
from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.internet.ssl import (
AcceptableCiphers,
CertificateOptions,
optionsForClientTLS,
platformTrust,
)
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
from scrapy.utils.misc import create_instance, load_object
from scrapy.core.downloader.tls import (
DEFAULT_CIPHERS,
ScrapyClientTLSOptions,
openssl_methods,
)
from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from twisted.internet._sslverify import ClientTLSOptions
@implementer(IPolicyForHTTPS)
@ -24,22 +38,44 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
def __init__(
self,
method: int = SSL.SSLv23_METHOD,
tls_verbose_logging: bool = False,
tls_ciphers: Optional[str] = None,
*args: Any,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
self._ssl_method: int = method
self.tls_verbose_logging: bool = tls_verbose_logging
self.tls_ciphers: AcceptableCiphers
if tls_ciphers:
self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
else:
self.tls_ciphers = DEFAULT_CIPHERS
@classmethod
def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
def from_settings(
cls,
settings: BaseSettings,
method: int = SSL.SSLv23_METHOD,
*args: Any,
**kwargs: Any,
):
tls_verbose_logging: bool = settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
return cls( # type: ignore[misc]
method=method,
tls_verbose_logging=tls_verbose_logging,
tls_ciphers=tls_ciphers,
*args,
**kwargs,
)
def getCertificateOptions(self):
def getCertificateOptions(self) -> CertificateOptions:
# setting verify=True will require you to provide CAs
# to verify against; in other words: it's not that simple
@ -53,19 +89,24 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
method=getattr(self, "method", getattr(self, "_ssl_method", None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
)
# kept for old-style HTTP/1.0 downloader context twisted calls,
# e.g. connectSSL()
def getContext(self, hostname=None, port=None):
return self.getCertificateOptions().getContext()
def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context:
ctx = self.getCertificateOptions().getContext()
ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT
return ctx
def creatorForNetloc(self, hostname, port):
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
verbose_logging=self.tls_verbose_logging)
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
return ScrapyClientTLSOptions(
hostname.decode("ascii"),
self.getContext(),
verbose_logging=self.tls_verbose_logging,
)
@implementer(IPolicyForHTTPS)
@ -87,7 +128,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
def creatorForNetloc(self, hostname, port):
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@ -95,7 +136,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
return optionsForClientTLS(
hostname=hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
extraCertificateOptions={"method": self._ssl_method},
)
@ -106,34 +147,34 @@ class AcceptableProtocolsContextFactory:
negotiation.
"""
def __init__(self, context_factory, acceptable_protocols):
def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]):
verifyObject(IPolicyForHTTPS, context_factory)
self._wrapped_context_factory = context_factory
self._acceptable_protocols = acceptable_protocols
self._wrapped_context_factory: Any = context_factory
self._acceptable_protocols: List[bytes] = acceptable_protocols
def creatorForNetloc(self, hostname, port):
options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(
hostname, port
)
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)
return options
def load_context_factory_from_settings(settings, crawler):
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
# try method-aware context factory
try:
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
context_factory = build_from_crawler(
context_factory_cls,
crawler,
method=ssl_method,
)
except TypeError:
# use context factory defaults
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
context_factory = build_from_crawler(
context_factory_cls,
crawler,
)
msg = (
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "

62
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,35 +1,41 @@
"""Download handlers for different schemes"""
import logging
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast
from twisted.internet import defer
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class DownloadHandlers:
def __init__(self, crawler):
self._crawler = crawler
self._schemes = {} # stores acceptable schemes on instancing
self._handlers = {} # stores instanced handlers for schemes
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
def __init__(self, crawler: "Crawler"):
self._crawler: "Crawler" = crawler
self._schemes: Dict[
str, Union[str, Callable]
] = {} # stores acceptable schemes on instancing
self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes
self._notconfigured: Dict[str, str] = {} # remembers failed handlers
handlers: Dict[str, Union[str, Callable]] = without_none_values(
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
)
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
crawler.signals.connect(self._close, signals.engine_stopped)
def _get_handler(self, scheme):
def _get_handler(self, scheme: str) -> Any:
"""Lazy-load the downloadhandler for a scheme
only on the first request for that scheme.
"""
@ -38,44 +44,48 @@ class DownloadHandlers:
if scheme in self._notconfigured:
return None
if scheme not in self._schemes:
self._notconfigured[scheme] = 'no handler available for that scheme'
self._notconfigured[scheme] = "no handler available for that scheme"
return None
return self._load_handler(scheme)
def _load_handler(self, scheme, skip_lazy=False):
def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any:
path = self._schemes[scheme]
try:
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, 'lazy', True):
if skip_lazy and getattr(dhcls, "lazy", True):
return None
dh = create_instance(
objcls=dhcls,
settings=self._crawler.settings,
crawler=self._crawler,
dh = build_from_crawler(
dhcls,
self._crawler,
)
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)
return None
except Exception as ex:
logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True, extra={'crawler': self._crawler})
logger.error(
'Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True,
extra={"crawler": self._crawler},
)
self._notconfigured[scheme] = str(ex)
return None
else:
self._handlers[scheme] = dh
return dh
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
return handler.download_request(request, spider)
raise NotSupported(
f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
)
return cast(Deferred, handler.download_request(request, spider))
@defer.inlineCallbacks
def _close(self, *_a, **_kw):
def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]:
for dh in self._handlers.values():
if hasattr(dh, 'close'):
if hasattr(dh, "close"):
yield dh.close()

16
scrapy/core/downloader/handlers/datauri.py
View File

@ -1,6 +1,9 @@
from typing import Any, Dict
from w3lib.url import parse_data_uri
from scrapy.http import TextResponse
from scrapy import Request, Spider
from scrapy.http import Response, TextResponse
from scrapy.utils.decorators import defers
from scrapy.utils.response import get_response_class
@ -9,17 +12,16 @@ class DataURIDownloadHandler:
lazy = False
@defers
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Response:
uri = parse_data_uri(request.url)
respcls = get_response_class(
body=uri.data,
declared_mime_types=(uri.media_type.encode(),),
)
resp_kwargs = {}
if (issubclass(respcls, TextResponse)
and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
resp_kwargs: Dict[str, Any] = {}
if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
charset = uri.media_type_parameters.get("charset")
resp_kwargs["encoding"] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)

22
scrapy/core/downloader/handlers/ftp.py
View File

@ -71,9 +71,9 @@ class FTPDownloadHandler:
}
def __init__(self, settings):
self.default_user = settings['FTP_USER']
self.default_password = settings['FTP_PASSWORD']
self.passive_mode = settings['FTP_PASSIVE_MODE']
self.default_user = settings["FTP_USER"]
self.default_password = settings["FTP_PASSWORD"]
self.passive_mode = settings["FTP_PASSIVE_MODE"]
@classmethod
def from_crawler(cls, crawler):
@ -81,12 +81,16 @@ class FTPDownloadHandler:
def download_request(self, request, spider):
from twisted.internet import reactor
parsed_url = urlparse_cached(request)
user = request.meta.get("ftp_user", self.default_user)
password = request.meta.get("ftp_password", self.default_password)
passive_mode = 1 if bool(request.meta.get("ftp_passive",
self.passive_mode)) else 0
creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
passive_mode = (
1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0
)
creator = ClientCreator(
reactor, FTPClient, user, password, passive=passive_mode
)
dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
@ -103,7 +107,7 @@ class FTPDownloadHandler:
def _build_response(self, result, request, protocol):
self.result = result
protocol.close()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
headers = {"local filename": protocol.filename or "", "size": protocol.size}
body = to_bytes(protocol.filename or protocol.body.read())
respcls = get_response_class(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
@ -115,5 +119,7 @@ class FTPDownloadHandler:
if m:
ftpcode = m.group()
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
return Response(url=request.url, status=httpcode, body=to_bytes(message))
return Response(
url=request.url, status=httpcode, body=to_bytes(message)
)
raise result.type(result.value)

18
scrapy/core/downloader/handlers/http10.py
View File

@ -1,6 +1,6 @@
"""Download handlers for http and https schemes
"""
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
@ -8,8 +8,10 @@ class HTTP10DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
self.ClientContextFactory = load_object(
settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
self._settings = settings
self._crawler = crawler
@ -25,12 +27,12 @@ class HTTP10DownloadHandler:
def _connect(self, factory):
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
client_context_factory = create_instance(
objcls=self.ClientContextFactory,
settings=self._settings,
crawler=self._crawler,
if factory.scheme == b"https":
client_context_factory = build_from_crawler(
self.ClientContextFactory,
self._crawler,
)
return reactor.connectSSL(host, port, factory, client_context_factory)
return reactor.connectTCP(host, port, factory)

264
scrapy/core/downloader/handlers/http11.py
View File

@ -12,10 +12,16 @@ from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.python.failure import Failure
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import (
URI,
Agent,
HTTPConnectionPool,
ResponseDone,
ResponseFailed,
)
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
from zope.interface import implementer
from scrapy import signals
@ -36,14 +42,17 @@ class HTTP11DownloadHandler:
self._crawler = crawler
from twisted.internet import reactor
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool.maxPersistentPerHost = settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self._pool._factory.noisy = False
self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE")
self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
self._disconnect_timeout = 1
@classmethod
@ -55,8 +64,8 @@ class HTTP11DownloadHandler:
agent = ScrapyAgent(
contextFactory=self._contextFactory,
pool=self._pool,
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
fail_on_dataloss=self._fail_on_dataloss,
crawler=self._crawler,
)
@ -64,6 +73,7 @@ class HTTP11DownloadHandler:
def close(self):
from twisted.internet import reactor
d = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
@ -96,11 +106,23 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
with this endpoint comes from the pool and a CONNECT has already been issued
for it.
"""
_truncatedLength = 1000
_responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
_responseAnswer = (
r"HTTP/1\.. (?P<status>\d{3})(?P<reason>.{," + str(_truncatedLength) + r"})"
)
_responseMatcher = re.compile(_responseAnswer.encode())
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
def __init__(
self,
reactor,
host,
port,
proxyConf,
contextFactory,
timeout=30,
bindAddress=None,
):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
@ -111,7 +133,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol):
"""Asks the proxy to open a tunnel."""
tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)
tunnelReq = tunnel_request_data(
self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
@ -129,24 +153,30 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
# from the proxy so that we don't send those bytes to the TLS layer
#
# see https://github.com/scrapy/scrapy/issues/2491
if b'\r\n\r\n' not in self._connectBuffer:
if b"\r\n\r\n" not in self._connectBuffer:
return
self._protocol.dataReceived = self._protocolDataReceived
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group('status')) == 200:
if respm and int(respm.group("status")) == 200:
# set proper Server Name Indication extension
sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
sslOptions = self._contextFactory.creatorForNetloc(
self._tunneledHost, self._tunneledPort
)
self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
if respm:
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
extra = {
"status": int(respm.group("status")),
"reason": respm.group("reason").strip(),
}
else:
extra = rcvd_bytes[:self._truncatedLength]
extra = rcvd_bytes[: self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')
TunnelError(
"Could not open CONNECT tunnel with proxy "
f"{self._host}:{self._port} [{extra!r}]"
)
)
def connectFailed(self, reason):
@ -173,12 +203,12 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
>>> s(tunnel_request_data(b"example.com", "8090"))
'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
"""
host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
tunnel_req += b'Host: ' + host_value + b'\r\n'
host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
tunnel_req += b"Host: " + host_value + b"\r\n"
if proxy_auth_header:
tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
tunnel_req += b'\r\n'
tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n"
tunnel_req += b"\r\n"
return tunnel_req
@ -190,8 +220,15 @@ class TunnelingAgent(Agent):
proxy involved.
"""
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
def __init__(
self,
reactor,
proxyConf,
contextFactory=None,
connectTimeout=None,
bindAddress=None,
pool=None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@ -207,7 +244,9 @@ class TunnelingAgent(Agent):
bindAddress=self._endpointFactory._bindAddress,
)
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
def _requestWithEndpoint(
self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
):
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
@ -224,8 +263,9 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
def __init__(
self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None
):
super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
@ -252,13 +292,21 @@ class ScrapyProxyAgent(Agent):
class ScrapyAgent:
_Agent = Agent
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
def __init__(
self,
contextFactory=None,
connectTimeout=10,
bindAddress=None,
pool=None,
maxsize=0,
warnsize=0,
fail_on_dataloss=True,
crawler=None,
):
self._contextFactory = contextFactory
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
@ -271,14 +319,15 @@ class ScrapyAgent:
def _get_agent(self, request, timeout):
from twisted.internet import reactor
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
bindaddress = request.meta.get("bindaddress") or self._bindAddress
proxy = request.meta.get("proxy")
if proxy:
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
if scheme == b'https':
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
if scheme == b"https":
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
return self._TunnelingAgent(
reactor=reactor,
@ -288,11 +337,11 @@ class ScrapyAgent:
bindAddress=bindaddress,
pool=self._pool,
)
proxyScheme = proxyScheme or b'http'
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
proxyScheme = proxyScheme or b"http"
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", ""))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding='ascii'),
proxyURI=to_bytes(proxyURI, encoding="ascii"),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
@ -308,7 +357,8 @@ class ScrapyAgent:
def download_request(self, request):
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connectTimeout
timeout = request.meta.get("download_timeout") or self._connectTimeout
agent = self._get_agent(request, timeout)
# request details
@ -316,13 +366,15 @@ class ScrapyAgent:
method = to_bytes(request.method)
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
headers.removeHeader(b'Proxy-Authorization')
headers.removeHeader(b"Proxy-Authorization")
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
else:
bodyproducer = None
start_time = time()
d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
d = agent.request(
method, to_bytes(url, encoding="ascii"), headers, bodyproducer
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
@ -345,14 +397,14 @@ class ScrapyAgent:
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time
request.meta["download_latency"] = time() - start_time
return result
@staticmethod
def _headers_from_twisted_response(response):
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b'Content-Length'] = str(response.length).encode()
headers[b"Content-Length"] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
@ -366,8 +418,10 @@ class ScrapyAgent:
)
for handler, result in headers_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__})
logger.debug(
"Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__},
)
txresponse._transport.stopProducing()
txresponse._transport.loseConnection()
return {
@ -389,15 +443,23 @@ class ScrapyAgent:
"ip_address": None,
}
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
maxsize = request.meta.get("download_maxsize", self._maxsize)
warnsize = request.meta.get("download_warnsize", self._warnsize)
expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
fail_on_dataloss = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if maxsize and expected_size > maxsize:
warning_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
warning_msg = (
"Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s)."
)
warning_args = {
"url": request.url,
"size": expected_size,
"maxsize": maxsize,
}
logger.warning(warning_msg, warning_args)
@ -405,9 +467,11 @@ class ScrapyAgent:
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
"download warn size (%(warnsize)s) in request %(request)s.",
{'size': expected_size, 'warnsize': warnsize, 'request': request})
logger.warning(
"Expected response size (%(size)s) larger than "
"download warn size (%(warnsize)s) in request %(request)s.",
{"size": expected_size, "warnsize": warnsize, "request": request},
)
def _cancel(_):
# Abort connection immediately.
@ -457,7 +521,6 @@ class ScrapyAgent:
@implementer(IBodyProducer)
class _RequestBodyProducer:
def __init__(self, body):
self.body = body
self.length = len(body)
@ -474,8 +537,16 @@ class _RequestBodyProducer:
class _ResponseReader(protocol.Protocol):
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
def __init__(
self,
finished,
txresponse,
request,
maxsize,
warnsize,
fail_on_dataloss,
crawler,
):
self._finished = finished
self._txresponse = txresponse
self._request = request
@ -491,22 +562,28 @@ class _ResponseReader(protocol.Protocol):
self._crawler = crawler
def _finish_response(self, flags=None, failure=None):
self._finished.callback({
"txresponse": self._txresponse,
"body": self._bodybuf.getvalue(),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"failure": failure,
})
self._finished.callback(
{
"txresponse": self._txresponse,
"body": self._bodybuf.getvalue(),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"failure": failure,
}
)
def connectionMade(self):
if self._certificate is None:
with suppress(AttributeError):
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
self._certificate = ssl.Certificate(
self.transport._producer.getPeerCertificate()
)
if self._ip_address is None:
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
self._ip_address = ipaddress.ip_address(
self.transport._producer.getPeer().host
)
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered data.
@ -524,29 +601,40 @@ class _ResponseReader(protocol.Protocol):
)
for handler, result in bytes_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
logger.debug(
"Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__},
)
self.transport.stopProducing()
self.transport.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.warning("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
logger.warning(
"Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{
"bytes": self._bytes_received,
"maxsize": self._maxsize,
"request": self._request,
},
)
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
if (
self._warnsize
and self._bytes_received > self._warnsize
and not self._reached_warnsize
):
self._reached_warnsize = True
logger.warning("Received more bytes than download "
"warn size (%(warnsize)s) in request %(request)s.",
{'warnsize': self._warnsize,
'request': self._request})
logger.warning(
"Received more bytes than download "
"warn size (%(warnsize)s) in request %(request)s.",
{"warnsize": self._warnsize, "request": self._request},
)
def connectionLost(self, reason):
if self._finished.called:
@ -560,16 +648,20 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["partial"])
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if reason.check(ResponseFailed) and any(
r.check(_DataLoss) for r in reason.value.reasons
):
if not self._fail_on_dataloss:
self._finish_response(flags=["dataloss"])
return
if not self._fail_on_dataloss_warned:
logger.warning("Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",
self._txresponse.request.absoluteURI.decode())
logger.warning(
"Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",
self._txresponse.request.absoluteURI.decode(),
)
self._fail_on_dataloss_warned = True
self._finished.errback(reason)

39
scrapy/core/downloader/handlers/http2.py
View File

@ -16,8 +16,9 @@ from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
H2DownloadHandlerOrSubclass = TypeVar(
"H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
)
class H2DownloadHandler:
@ -25,11 +26,14 @@ class H2DownloadHandler:
self._crawler = crawler
from twisted.internet import reactor
self._pool = H2ConnectionPool(reactor, settings)
self._context_factory = load_context_factory_from_settings(settings, crawler)
@classmethod
def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
def from_crawler(
cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler
) -> H2DownloadHandlerOrSubclass:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
@ -49,7 +53,8 @@ class ScrapyH2Agent:
_ProxyAgent = ScrapyProxyH2Agent
def __init__(
self, context_factory,
self,
context_factory,
pool: H2ConnectionPool,
connect_timeout: int = 10,
bind_address: Optional[bytes] = None,
@ -63,19 +68,22 @@ class ScrapyH2Agent:
def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
from twisted.internet import reactor
bind_address = request.meta.get('bindaddress') or self._bind_address
proxy = request.meta.get('proxy')
bind_address = request.meta.get("bindaddress") or self._bind_address
proxy = request.meta.get("proxy")
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
if scheme == b'https':
if scheme == b"https":
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
raise NotImplementedError(
"Tunneling via CONNECT method using HTTP/2.0 is not yet supported"
)
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
@ -91,7 +99,8 @@ class ScrapyH2Agent:
def download_request(self, request: Request, spider: Spider) -> Deferred:
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connect_timeout
timeout = request.meta.get("download_timeout") or self._connect_timeout
agent = self._get_agent(request, timeout)
start_time = time()
@ -103,12 +112,16 @@ class ScrapyH2Agent:
return d
@staticmethod
def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
request.meta['download_latency'] = time() - start_time
def _cb_latency(
response: Response, request: Request, start_time: float
) -> Response:
request.meta["download_latency"] = time() - start_time
return response
@staticmethod
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
def _cb_timeout(
response: Response, request: Request, timeout: float, timeout_cl: DelayedCall
) -> Response:
if timeout_cl.active():
timeout_cl.cancel()
return response

73
scrapy/core/downloader/handlers/s3.py
View File

@ -2,49 +2,57 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
def __init__(
self,
settings,
*,
crawler=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler,
**kw,
):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
raise NotConfigured("missing botocore library")
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
aws_access_key_id = settings["AWS_ACCESS_KEY_ID"]
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"]
if not aws_session_token:
aws_session_token = settings['AWS_SESSION_TOKEN']
aws_session_token = settings["AWS_SESSION_TOKEN"]
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
# unless 'anon' was set explicitly (True/False).
anon = kw.get('anon')
anon = kw.get("anon")
if anon is None and not aws_access_key_id and not aws_secret_access_key:
kw['anon'] = True
self.anon = kw.get('anon')
kw["anon"] = True
self.anon = kw.get("anon")
self._signer = None
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key, aws_session_token))
_http_handler = create_instance(
objcls=httpdownloadhandler,
settings=settings,
crawler=crawler,
kw.pop("anon", None)
if kw:
raise TypeError(f"Unexpected keyword arguments: {kw}")
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
self._signer = SignerCls(
botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key, aws_session_token
)
)
_http_handler = build_from_crawler(
httpdownloadhandler,
crawler,
)
self._download_http = _http_handler.download_request
@ -54,20 +62,21 @@ class S3DownloadHandler:
def download_request(self, request, spider):
p = urlparse_cached(request)
scheme = 'https' if request.meta.get('is_secure') else 'http'
scheme = "https" if request.meta.get("is_secure") else "http"
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
path = p.path + "?" + p.query if p.query else p.path
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
if self.anon:
request = request.replace(url=url)
else:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
headers=request.headers.to_unicode_dict(),
data=request.body)
data=request.body,
)
self._signer.add_auth(awsrequest)
request = request.replace(
url=url, headers=awsrequest.headers.items())
request = request.replace(url=url, headers=awsrequest.headers.items())
return self._download_http(request, spider)

Some files were not shown because too many files have changed in this diff Show More