RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into case-insensitive-dict

This commit is contained in:
Andrey Rakhmatullin 2023-06-21 22:12:44 +04:00
parent 2c65066ad9 e9e1034af3
commit 39282d7bd0
414 changed files with 27454 additions and 16384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bandit.yml
View File

@ -1,5 +1,6 @@
skips:
- B101
- B113 # https://github.com/PyCQA/bandit/issues/1010
- B105
- B301
- B303
@ -17,3 +18,4 @@ skips:
- B503
- B603
- B605
exclude_dirs: ['tests']

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.6.1
current_version = 2.9.0
commit = True
tag = True
tag_name = {new_version}

1
.coveragerc
View File

@ -3,3 +3,4 @@ branch = true
include = scrapy/*
omit =
tests/*
disable_warnings = include-ignored

21
.flake8
View File

@ -1,19 +1,22 @@
[flake8]
max-line-length = 119
ignore = W503
ignore = W503, E203
exclude =
docs/conf.py
per-file-ignores =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
scrapy/__init__.py:E402
scrapy/core/downloader/handlers/http.py:F401
scrapy/http/__init__.py:F401
scrapy/linkextractors/__init__.py:E402,F401
scrapy/selector/__init__.py:F401
scrapy/spiders/__init__.py:E402,F401
# Issues pending a review:
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
scrapy/utils/url.py:F403,F405
tests/test_loader.py:E741

7
.git-blame-ignore-revs Normal file
View File

@ -0,0 +1,7 @@
# .git-blame-ignore-revs
# adding black formatter to all the code
e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
# re applying black to the code with default line length
303f0a70fcf8067adf0a909c2096a5009162383a
# reaplying black again and removing line length on pre-commit black config
c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962

29
.github/workflows/checks.yml vendored
View File

@ -3,34 +3,29 @@ on: [push, pull_request]
jobs:
checks:
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: "3.10"
env:
TOXENV: security
- python-version: "3.10"
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
- python-version: "3.11"
env:
TOXENV: pylint
- python-version: 3.6
- python-version: 3.8
env:
TOXENV: typing
- python-version: "3.10" # Keep in sync with .readthedocs.yml
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
- python-version: "3.11"
env:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
@ -39,3 +34,9 @@ jobs:
run: |
pip install -U tox
tox
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: pre-commit/action@v3.0.0

42
.github/workflows/publish.yml vendored
View File

@ -1,31 +1,21 @@
name: Publish
on: [push]
on:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
jobs:
publish:
runs-on: ubuntu-18.04
if: startsWith(github.event.ref, 'refs/tags/')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Check Tag
id: check-release-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
echo ::set-output name=release_tag::true
fi
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.11
- run: |
pip install --upgrade build twine
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@v1.6.4
with:
password: ${{ secrets.PYPI_TOKEN }}

8
.github/workflows/tests-macos.yml vendored
View File

@ -3,17 +3,17 @@ on: [push, pull_request]
jobs:
tests:
runs-on: macos-10.15
runs-on: macos-11
strategy:
fail-fast: false
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
python-version: ["3.8", "3.9", "3.10", "3.11"]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

53
.github/workflows/tests-ubuntu.yml vendored
View File

@ -3,75 +3,68 @@ on: [push, pull_request]
jobs:
tests:
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.11"
env:
TOXENV: asyncio
- python-version: pypy3
- python-version: pypy3.9
env:
TOXENV: pypy3
PYPY_VERSION: 3.6-v7.3.3
# pinned deps
- python-version: 3.6.12
- python-version: 3.8.17
env:
TOXENV: pinned
- python-version: 3.6.12
- python-version: 3.8.17
env:
TOXENV: asyncio-pinned
- python-version: pypy3
- python-version: pypy3.8
env:
TOXENV: pypy3-pinned
PYPY_VERSION: 3.6-v7.2.0
- python-version: 3.8.17
env:
TOXENV: extra-deps-pinned
- python-version: 3.8.17
env:
TOXENV: botocore-pinned
# extras
# extra-deps includes reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
- python-version: "3.11"
env:
TOXENV: extra-deps
- python-version: "3.11"
env:
TOXENV: botocore
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
# libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
sudo apt-get install libxml2-dev libxslt-dev
- name: Run tests
env: ${{ matrix.env }}
run: |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
pip install -U tox
tox

19
.github/workflows/tests-windows.yml vendored
View File

@ -8,15 +8,9 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.6
env:
TOXENV: windows-pinned
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
TOXENV: windows-pinned
- python-version: 3.9
env:
TOXENV: py
@ -26,12 +20,19 @@ jobs:
- python-version: "3.10"
env:
TOXENV: asyncio
# no binary package for lxml for 3.11 yet
# - python-version: "3.11"
# env:
# TOXENV: py
# - python-version: "3.11"
# env:
# TOXENV: asyncio
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

3
.gitignore vendored
View File

@ -23,3 +23,6 @@ test-output.*
# Windows
Thumbs.db
# OSX miscellaneous
.DS_Store

2
.isort.cfg Normal file
View File

@ -0,0 +1,2 @@
[settings]
profile = black

24
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,24 @@
repos:
- repo: https://github.com/PyCQA/bandit
rev: 1.7.5
hooks:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 6.0.0
hooks:
- id: flake8
- repo: https://github.com/psf/black.git
rev: 23.3.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.13.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==23.3.0

2
.readthedocs.yml
View File

@ -9,7 +9,7 @@ build:
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: "3.10" # Keep in sync with .github/workflows/checks.yml
python: "3.11" # Keep in sync with .github/workflows/checks.yml
python:
install:

152
CODE_OF_CONDUCT.md
View File

@ -1,77 +1,133 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
Examples of behavior that contributes to a positive environment for our
community include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior by participants include:
Examples of unacceptable behavior include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
## Enforcement Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at opensource@zyte.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
reported to the community leaders responsible for enforcement at
opensource@zyte.com.
All complaints will be reviewed and investigated promptly and fairly.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version].
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations

4
INSTALL
View File

@ -1,4 +0,0 @@
For information about installing Scrapy see:
* docs/intro/install.rst (local file)
* https://docs.scrapy.org/en/latest/intro/install.html (online version)

4
INSTALL.md Normal file
View File

@ -0,0 +1,4 @@
For information about installing Scrapy see:
* [Local docs](docs/intro/install.rst)
* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)

10
README.rst
View File

@ -1,4 +1,5 @@
.. image:: https://scrapy.org/img/scrapylogo.png
:target: https://scrapy.org/
======
Scrapy
@ -57,13 +58,15 @@ including a list of features.
Requirements
============
* Python 3.6+
* Python 3.8+
* Works on Linux, Windows, macOS, BSD
Install
=======
The quick way::
The quick way:
.. code:: bash
pip install scrapy
@ -94,8 +97,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
Code of Conduct
---------------
Please note that this project is released with a Contributor Code of Conduct
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.
By participating in this project you agree to abide by its terms.
Please report unacceptable behavior to opensource@zyte.com.

8
artwork/README.rst
View File

@ -2,19 +2,19 @@
Scrapy artwork
==============
This folder contains Scrapy artwork resources such as logos and fonts.
This folder contains the Scrapy artwork resources such as logos and fonts.
scrapy-logo.jpg
---------------
Main Scrapy logo, in JPEG format.
The main Scrapy logo, in JPEG format.
qlassik.zip
-----------
Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
scrapy-blog.logo.xcf
--------------------
The logo used in Scrapy blog, in Gimp format.
The logo used in the Scrapy blog, in Gimp format.

32
conftest.py
View File

@ -4,12 +4,11 @@ import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
return (str(p) for p in Path(folder).rglob("*.py"))
collect_ignore = [
@ -21,16 +20,16 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
with open('tests/ignores.txt') as reader:
with Path("tests/ignores.txt").open(encoding="utf-8") as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
if file_path and file_path[0] != "#":
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
"scrapy/core/downloader/handlers/http2.py",
*_py_files("scrapy/core/http2"),
)
)
@ -42,16 +41,6 @@ def chdir(tmpdir):
tmpdir.chdir()
def pytest_collection_modifyitems(session, config, items):
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
try:
from pytest_flake8 import Flake8Item
if config.getoption('--flake8'):
items[:] = [item for item in items if isinstance(item, Flake8Item)]
except ImportError:
pass
def pytest_addoption(parser):
parser.addoption(
"--reactor",
@ -60,7 +49,7 @@ def pytest_addoption(parser):
)
@pytest.fixture(scope='class')
@pytest.fixture(scope="class")
def reactor_pytest(request):
if not request.cls:
# doctests
@ -71,14 +60,17 @@ def reactor_pytest(request):
@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
pytest.skip("This test is only run with --reactor=asyncio")
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
pytest.skip('This test is only run without --reactor=asyncio')
if (
request.node.get_closest_marker("only_not_asyncio")
and reactor_pytest == "asyncio"
):
pytest.skip("This test is only run without --reactor=asyncio")
def pytest_configure(config):

4
docs/Makefile
View File

@ -86,8 +86,8 @@ coverage: BUILDER = coverage
coverage: build
htmlview: html
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
$(PYTHON) -c "import webbrowser; from pathlib import Path; \
webbrowser.open(Path('build/html/index.html').resolve().as_uri())"
clean:
-rm -rf build/*

2
docs/README.rst
View File

@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
Start over
----------
To cleanup all generated documentation files and start from scratch run::
To clean up all generated documentation files and start from scratch run::
make clean

105
docs/_ext/scrapydocs.py
View File

@ -1,8 +1,9 @@
from docutils.parsers.rst.roles import set_classes
from operator import itemgetter
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst.roles import set_classes
from sphinx.util.nodes import make_refnode
from operator import itemgetter
class settingslist_node(nodes.General, nodes.Element):
@ -11,15 +12,15 @@ class settingslist_node(nodes.General, nodes.Element):
class SettingsListDirective(Directive):
def run(self):
return [settingslist_node('')]
return [settingslist_node("")]
def is_setting_index(node):
if node.tagname == 'index':
if node.tagname == "index" and node["entries"]:
# index entries for setting directives look like:
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
entry_type, info, refid = node["entries"][0][:3]
return entry_type == "pair" and info.endswith("; setting")
return False
@ -30,14 +31,14 @@ def get_setting_target(node):
def get_setting_name_and_refid(node):
"""Extract setting name from directive index node"""
entry_type, info, refid = node['entries'][0][:3]
return info.replace('; setting', ''), refid
entry_type, info, refid = node["entries"][0][:3]
return info.replace("; setting", ""), refid
def collect_scrapy_settings_refs(app, doctree):
env = app.builder.env
if not hasattr(env, 'scrapy_all_settings'):
if not hasattr(env, "scrapy_all_settings"):
env.scrapy_all_settings = []
for node in doctree.traverse(is_setting_index):
@ -46,18 +47,23 @@ def collect_scrapy_settings_refs(app, doctree):
setting_name, refid = get_setting_name_and_refid(node)
env.scrapy_all_settings.append({
'docname': env.docname,
'setting_name': setting_name,
'refid': refid,
})
env.scrapy_all_settings.append(
{
"docname": env.docname,
"setting_name": setting_name,
"refid": refid,
}
)
def make_setting_element(setting_data, app, fromdocname):
refnode = make_refnode(app.builder, fromdocname,
todocname=setting_data['docname'],
targetid=setting_data['refid'],
child=nodes.Text(setting_data['setting_name']))
refnode = make_refnode(
app.builder,
fromdocname,
todocname=setting_data["docname"],
targetid=setting_data["refid"],
child=nodes.Text(setting_data["setting_name"]),
)
p = nodes.paragraph()
p += refnode
@ -71,69 +77,72 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
for node in doctree.traverse(settingslist_node):
settings_list = nodes.bullet_list()
settings_list.extend([make_setting_element(d, app, fromdocname)
for d in sorted(env.scrapy_all_settings,
key=itemgetter('setting_name'))
if fromdocname != d['docname']])
settings_list.extend(
[
make_setting_element(d, app, fromdocname)
for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name"))
if fromdocname != d["docname"]
]
)
node.replace_self(settings_list)
def setup(app):
app.add_crossref_type(
directivename = "setting",
rolename = "setting",
indextemplate = "pair: %s; setting",
directivename="setting",
rolename="setting",
indextemplate="pair: %s; setting",
)
app.add_crossref_type(
directivename = "signal",
rolename = "signal",
indextemplate = "pair: %s; signal",
directivename="signal",
rolename="signal",
indextemplate="pair: %s; signal",
)
app.add_crossref_type(
directivename = "command",
rolename = "command",
indextemplate = "pair: %s; command",
directivename="command",
rolename="command",
indextemplate="pair: %s; command",
)
app.add_crossref_type(
directivename = "reqmeta",
rolename = "reqmeta",
indextemplate = "pair: %s; reqmeta",
directivename="reqmeta",
rolename="reqmeta",
indextemplate="pair: %s; reqmeta",
)
app.add_role('source', source_role)
app.add_role('commit', commit_role)
app.add_role('issue', issue_role)
app.add_role('rev', rev_role)
app.add_role("source", source_role)
app.add_role("commit", commit_role)
app.add_role("issue", issue_role)
app.add_role("rev", rev_role)
app.add_node(settingslist_node)
app.add_directive('settingslist', SettingsListDirective)
app.add_directive("settingslist", SettingsListDirective)
app.connect('doctree-read', collect_scrapy_settings_refs)
app.connect('doctree-resolved', replace_settingslist_nodes)
app.connect("doctree-read", collect_scrapy_settings_refs)
app.connect("doctree-resolved", replace_settingslist_nodes)
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
ref = "https://github.com/scrapy/scrapy/blob/master/" + text
set_classes(options)
node = nodes.reference(rawtext, text, refuri=ref, **options)
return [node], []
def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/issues/' + text
ref = "https://github.com/scrapy/scrapy/issues/" + text
set_classes(options)
node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options)
return [node], []
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/commit/' + text
ref = "https://github.com/scrapy/scrapy/commit/" + text
set_classes(options)
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options)
return [node], []
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
ref = "http://hg.scrapy.org/scrapy/changeset/" + text
set_classes(options)
node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)
node = nodes.reference(rawtext, "r" + text, refuri=ref, **options)
return [node], []

31
docs/_static/selectors-sample1.html vendored
View File

@ -1,16 +1,17 @@
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
</div>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
</div>
</body>
</html>

11
docs/_templates/layout.html vendored
View File

@ -1,11 +0,0 @@
{% extends "!layout.html" %}
{% block footer %}
{{ super() }}
<script type="text/javascript">
analytics.ready(function () {
ga('require', 'linker');
ga('linker:autoLink', ['zyte.com']);
});
</script>
{% endblock %}

184
docs/conf.py
View File

@ -11,13 +11,12 @@
import sys
from datetime import datetime
from os import path
from pathlib import Path
# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
sys.path.append(path.join(path.dirname(__file__), "_ext"))
sys.path.insert(0, path.dirname(path.dirname(__file__)))
# is relative to the documentation root, use Path.absolute to make it absolute.
sys.path.append(str(Path(__file__).parent / "_ext"))
sys.path.insert(0, str(Path(__file__).parent.parent))
# General configuration
@ -26,30 +25,30 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'hoverxref.extension',
'notfound.extension',
'scrapydocs',
'sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
"hoverxref.extension",
"notfound.extension",
"scrapydocs",
"sphinx.ext.autodoc",
"sphinx.ext.coverage",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8'
# source_encoding = 'utf-8'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'Scrapy'
copyright = f'2008–{datetime.now().year}, Scrapy developers'
project = "Scrapy"
copyright = f"2008–{datetime.now().year}, Scrapy developers"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -58,50 +57,51 @@ copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The short X.Y version.
try:
import scrapy
version = '.'.join(map(str, scrapy.version_info[:2]))
version = ".".join(map(str, scrapy.version_info[:2]))
release = scrapy.__version__
except ImportError:
version = ''
release = ''
version = ""
release = ""
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
language = 'en'
language = "en"
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of documents that shouldn't be included in the build.
#unused_docs = []
# unused_docs = []
exclude_patterns = ['build']
exclude_patterns = ["build"]
# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['.build']
exclude_trees = [".build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# List of Sphinx warnings that will not be raised
suppress_warnings = ['epub.unknown_project_files']
suppress_warnings = ["epub.unknown_project_files"]
# Options for HTML output
@ -109,17 +109,18 @@ suppress_warnings = ['epub.unknown_project_files']
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
# Add path to the RTD explicitly to robustify builds (otherwise might
# fail in a clean Debian build env)
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
@ -129,44 +130,44 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'
html_last_updated_fmt = "%b %d, %Y"
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_use_modindex = True
# html_use_modindex = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, the reST sources are included in the HTML build as _sources/<name>.
html_copy_source = True
@ -174,16 +175,16 @@ html_copy_source = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''
# html_file_suffix = ''
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
htmlhelp_basename = "Scrapydoc"
html_css_files = [
'custom.css',
"custom.css",
]
@ -191,34 +192,33 @@ html_css_files = [
# ------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', 'Scrapy Documentation',
'Scrapy developers', 'manual'),
("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_use_modindex = True
# latex_use_modindex = True
# Options for the linkcheck builder
@ -227,8 +227,9 @@ latex_documents = [
# A list of regular expressions that match URIs that should not be checked when
# doing a linkcheck build.
linkcheck_ignore = [
'http://localhost:\d+', 'http://hg.scrapy.org',
'http://directory.google.com/'
"http://localhost:\d+",
"http://hg.scrapy.org",
"http://directory.google.com/",
]
@ -238,44 +239,35 @@ coverage_ignore_pyobjects = [
# Contract’s add_pre_hook and add_post_hook are not documented because
# they should be transparent to contract developers, for whom pre_hook and
# post_hook should be the actual concern.
r'\bContract\.add_(pre|post)_hook$',
r"\bContract\.add_(pre|post)_hook$",
# ContractsManager is an internal class, developers are not expected to
# interact with it directly in any way.
r'\bContractsManager\b$',
r"\bContractsManager\b$",
# For default contracts we only want to document their general purpose in
# their __init__ method, the methods they reimplement to achieve that purpose
# should be irrelevant to developers using those contracts.
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
r"\w+Contract\.(adjust_request_args|(pre|post)_process)$",
# Methods of downloader middlewares are not documented, only the classes
# themselves, since downloader middlewares are controlled through Scrapy
# settings.
r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
r"^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.",
# Base classes of downloader middlewares are implementation details that
# are not meant for users.
r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware",
# Private exception used by the command-line interface implementation.
r'^scrapy\.exceptions\.UsageError',
r"^scrapy\.exceptions\.UsageError",
# Methods of BaseItemExporter subclasses are only documented in
# BaseItemExporter.
r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
r"^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.",
# Extension behavior is only modified through settings. Methods of
# extension classes, as well as helper functions, are implementation
# details that are not documented.
r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
r"^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.", # methods
r"^scrapy\.extensions\.[a-z]\w*?\.[a-z]", # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
r"^scrapy\.linkextractors\.FilteringLinkExtractor$",
# Implementation detail of LxmlLinkExtractor
r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
]
@ -283,18 +275,20 @@ coverage_ignore_pyobjects = [
# -------------------------------------
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cryptography' : ('https://cryptography.io/en/latest/', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
"attrs": ("https://www.attrs.org/en/stable/", None),
"coverage": ("https://coverage.readthedocs.io/en/stable", None),
"cryptography": ("https://cryptography.io/en/latest/", None),
"cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
"itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
"pytest": ("https://docs.pytest.org/en/latest", None),
"python": ("https://docs.python.org/3", None),
"sphinx": ("https://www.sphinx-doc.org/en/master", None),
"tox": ("https://tox.wiki/en/latest/", None),
"twisted": ("https://docs.twisted.org/en/stable/", None),
"twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
"w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
}
intersphinx_disabled_reftypes = []
# Options for sphinx-hoverxref options
@ -312,16 +306,16 @@ hoverxref_role_types = {
"setting": "tooltip",
"signal": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
def setup(app):
app.connect('autodoc-skip-member', maybe_skip_member)
app.connect("autodoc-skip-member", maybe_skip_member)
def maybe_skip_member(app, what, name, obj, skip, options):
if not skip:
# autodocs was generating a text "alias of" for the following members
# https://github.com/sphinx-doc/sphinx/issues/4422
return name in {'default_item_class', 'default_selector_class'}
return name in {"default_item_class", "default_selector_class"}
return skip

23
docs/conftest.py
View File

@ -1,33 +1,34 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
from scrapy.http.response.html import HtmlResponse
def load_response(url, filename):
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
with open(input_path, 'rb') as input_file:
return HtmlResponse(url, body=input_file.read())
def load_response(url: str, filename: str) -> HtmlResponse:
input_path = Path(__file__).parent / "_tests" / filename
return HtmlResponse(url, body=input_path.read_bytes())
def setup(namespace):
namespace['load_response'] = load_response
namespace["load_response"] = load_response
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
PythonCodeBlockParser(future_imports=['print_function']),
PythonCodeBlockParser(future_imports=["print_function"]),
skip,
],
pattern='*.rst',
pattern="*.rst",
setup=setup,
).pytest()

68
docs/contributing.rst
View File

@ -11,10 +11,6 @@ Contributing to Scrapy
There are many ways to contribute to Scrapy. Here are some of them:
* Blog about Scrapy. Tell the world how you're using Scrapy. This will help
newcomers with more examples and will help the Scrapy project to increase its
visibility.
* Report bugs and request features in the `issue tracker`_, trying to follow
the guidelines detailed in `Reporting bugs`_ below.
@ -22,13 +18,16 @@ There are many ways to contribute to Scrapy. Here are some of them:
:ref:`writing-patches` and `Submitting patches`_ below for details on how to
write and submit a patch.
* Blog about Scrapy. Tell the world how you're using Scrapy. This will help
newcomers with more examples and will help the Scrapy project to increase its
visibility.
* Join the `Scrapy subreddit`_ and share your ideas on how to
improve Scrapy. We're always open to suggestions.
* Answer Scrapy questions at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__.
Reporting bugs
==============
@ -49,7 +48,7 @@ guidelines when you're going to report a new bug.
(use "scrapy" tag).
* check the `open issues`_ to see if the issue has already been reported. If it
has, don't dismiss the report, but check the ticket history and comments. If
has, don't dismiss the report, but check the ticket history and comments. If
you have additional useful information, please leave a comment, or consider
:ref:`sending a pull request <writing-patches>` with a fix.
@ -80,6 +79,13 @@ guidelines when you're going to report a new bug.
Writing patches
===============
Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you
can work on. These issues are a great way to get started with contributing to
Scrapy. If you're new to the codebase, you may want to focus on documentation
or testing-related issues, as they are always useful and can help you get
more familiar with the project. You can also check Scrapy's `test coverage`_
to see which areas may benefit from more tests.
The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged.
Well-written patches should:
@ -169,16 +175,43 @@ Coding style
Please follow these coding conventions when writing code for inclusion in
Scrapy:
* Unless otherwise specified, follow :pep:`8`.
* It's OK to use lines longer than 79 chars if it improves the code
readability.
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
run black manually with ``tox -e black``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
.. _scrapy-pre-commit:
Pre-commit
==========
We use `pre-commit`_ to automatically address simple code issues before every
commit.
.. _pre-commit: https://pre-commit.com/
After your create a local clone of your fork of the Scrapy repository:
#. `Install pre-commit <https://pre-commit.com/#installation>`_.
#. On the root of your local clone of the Scrapy repository, run the following
command:
.. code-block:: bash
pre-commit install
Now pre-commit will check your changes every time you create a Git commit. Upon
finding issues, pre-commit aborts your commit, and either fixes those issues
automatically, or only reports them to you. If it fixes those issues
automatically, creating your commit again should succeed. Otherwise, you may
need to address the corresponding issues manually first.
.. _documentation-policies:
Documentation policies
@ -214,7 +247,7 @@ Tests
=====
Tests are implemented using the :doc:`Twisted unit-testing framework
<twisted:core/development/policy/test-standard>`. Running tests requires
<twisted:development/test-standard>`. Running tests requires
:doc:`tox <tox:index>`.
.. _running-tests:
@ -232,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
the tests with Python 3.6 use::
the tests with Python 3.10 use::
tox -e py36
tox -e py310
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py36,py38 -p auto
tox -e py39,py310 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@ -250,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
tox -e py36 -- scrapy tests -n auto
tox -e py310 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:
@ -287,3 +320,6 @@ And their unit-tests are in::
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy

55
docs/faq.rst
View File

@ -35,8 +35,10 @@ for parsing HTML responses in Scrapy callbacks.
You just have to feed the response's body into a ``BeautifulSoup`` object
and extract whatever data you need from it.
Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser::
Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:
.. skip: next
.. code-block:: python
from bs4 import BeautifulSoup
import scrapy
@ -45,17 +47,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
class ExampleSpider(scrapy.Spider):
name = "example"
allowed_domains = ["example.com"]
start_urls = (
'http://www.example.com/',
)
start_urls = ("http://www.example.com/",)
def parse(self, response):
# use lxml to get decent HTML parsing speed
soup = BeautifulSoup(response.text, 'lxml')
yield {
"url": response.url,
"title": soup.h1.string
}
soup = BeautifulSoup(response.text, "lxml")
yield {"url": response.url, "title": soup.h1.string}
.. note::
@ -109,11 +106,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient
in most cases.
If you do want to crawl in true `BFO order`_, you can do it by
setting the following settings::
setting the following settings:
.. code-block:: python
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
While pending requests are below the configured values of
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
@ -159,11 +158,13 @@ See also other suggestions at `StackOverflow`_.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation::
your custom implementation:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
'myproject.middlewares.CustomOffsiteMiddleware': 500,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
"myproject.middlewares.CustomOffsiteMiddleware": 500,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
@ -230,16 +231,20 @@ Can I return (Twisted) deferreds from signal handlers?
Some signals support returning deferreds from their handlers, others don't. See
the :ref:`topics-signals-ref` to know which ones.
What does the response status code 999 means?
---------------------------------------------
What does the response status code 999 mean?
--------------------------------------------
999 is a custom response status code used by Yahoo sites to throttle requests.
Try slowing down the crawling speed by using a download delay of ``2`` (or
higher) in your spider::
higher) in your spider:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class MySpider(CrawlSpider):
name = 'myspider'
name = "myspider"
download_delay = 2
@ -351,19 +356,21 @@ How to split an item into multiple items in an item pipeline?
input item. :ref:`Create a spider middleware <custom-spider-middleware>`
instead, and use its
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method for this purpose. For example::
method for this purpose. For example:
.. code-block:: python
from copy import deepcopy
from itemadapter import is_item, ItemAdapter
class MultiplyItemsMiddleware:
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if is_item(item):
adapter = ItemAdapter(item)
for _ in range(adapter['multiply_by']):
for _ in range(adapter["multiply_by"]):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?
@ -413,4 +420,4 @@ See :issue:`2680`.
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search

17
docs/index.rst
View File

@ -130,7 +130,6 @@ Built-in services
topics/stats
topics/email
topics/telnetconsole
topics/webservice
:doc:`topics/logging`
Learn how to use Python's builtin logging on Scrapy.
@ -144,9 +143,6 @@ Built-in services
:doc:`topics/telnetconsole`
Inspect a running crawler using a built-in Python console.
:doc:`topics/webservice`
Monitor and control a crawler using a web service.
Solving specific problems
=========================
@ -229,10 +225,11 @@ Extending Scrapy
topics/downloader-middleware
topics/spider-middleware
topics/extensions
topics/api
topics/signals
topics/scheduler
topics/exporters
topics/components
topics/api
:doc:`topics/architecture`
@ -247,9 +244,6 @@ Extending Scrapy
:doc:`topics/extensions`
Extend Scrapy with your custom functionality
:doc:`topics/api`
Use it on extensions and middlewares to extend Scrapy functionality
:doc:`topics/signals`
See all available signals and how to work with them.
@ -259,6 +253,13 @@ Extending Scrapy
:doc:`topics/exporters`
Quickly export your scraped items to a file (XML, CSV, etc).
:doc:`topics/components`
Learn the common API and some good practices when building custom Scrapy
components.
:doc:`topics/api`
Use it on extensions and middlewares to extend Scrapy functionality.
All the rest
============

20
docs/intro/install.rst
View File

@ -9,8 +9,8 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
Scrapy requires Python 3.8+, either the CPython implementation (default) or
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
@ -52,17 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
* `twisted`_, an asynchronous networking framework
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
The minimal versions which Scrapy is tested against are:
* Twisted 14.0
* lxml 3.4
* pyOpenSSL 0.14
Scrapy may work with older versions of these packages
but it is not guaranteed it will continue working
because it’s not being tested against them.
Some of these packages themselves depends on non-Python packages
Some of these packages themselves depend on non-Python packages
that might require additional installation steps depending on your platform.
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
@ -197,7 +187,7 @@ solutions:
* Install `homebrew`_ following the instructions in https://brew.sh/
* Update your ``PATH`` variable to state that homebrew packages should be
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
if you're using `zsh`_ as default shell)::
echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
@ -229,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy::
PyPy
----
We recommend using the latest PyPy version. The version tested is 5.9.0.
We recommend using the latest PyPy version.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.

18
docs/intro/overview.rst
View File

@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
example of a Scrapy Spider using the simplest way to run a spider.
Here's the code for a spider that scrapes famous quotes from website
https://quotes.toscrape.com, following the pagination::
https://quotes.toscrape.com, following the pagination:
.. code-block:: python
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/tag/humor/',
"https://quotes.toscrape.com/tag/humor/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'author': quote.xpath('span/small/text()').get(),
'text': quote.css('span.text::text').get(),
"author": quote.xpath("span/small/text()").get(),
"text": quote.css("span.text::text").get(),
}
next_page = response.css('li.next a::attr("href")').get()
@ -45,9 +47,9 @@ https://quotes.toscrape.com, following the pagination::
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.jl
scrapy runspider quotes_spider.py -o quotes.jsonl
When this finishes you will have in the ``quotes.jl`` file a list of the
When this finishes you will have in the ``quotes.jsonl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}

287
docs/intro/tutorial.rst
View File

@ -83,7 +83,11 @@ optionally how to follow links in the pages, and how to parse the downloaded
page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:
.. code-block:: python
from pathlib import Path
import scrapy
@ -93,18 +97,17 @@ This is the code for our first Spider. Save it in a file named
def start_requests(self):
urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
self.log(f'Saved file {filename}')
filename = f"quotes-{page}.html"
Path(filename).write_bytes(response.body)
self.log(f"Saved file {filename}")
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
@ -176,7 +179,11 @@ that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
for your spider.
.. code-block:: python
from pathlib import Path
import scrapy
@ -184,15 +191,14 @@ for your spider::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
filename = f"quotes-{page}.html"
Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy
@ -243,8 +249,10 @@ object:
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
.. code-block:: pycon
>>> response.css("title")
[<Selector query='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@ -254,42 +262,54 @@ data.
To extract the text from the title above, you can do:
>>> response.css('title::text').getall()
['Quotes to Scrape']
.. code-block:: pycon
>>> response.css("title::text").getall()
['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
CSS query, to mean we want to select only the text elements directly inside
``<title>`` element. If we don't specify ``::text``, we'd get the full title
element, including its tags:
>>> response.css('title').getall()
['<title>Quotes to Scrape</title>']
.. code-block:: pycon
>>> response.css("title").getall()
['<title>Quotes to Scrape</title>']
The other thing is that the result of calling ``.getall()`` is a list: it is
possible that a selector returns more than one result, so we extract them all.
When you know you just want the first result, as in this case, you can do:
>>> response.css('title::text').get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.css("title::text").get()
'Quotes to Scrape'
As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.css("title::text")[0].get()
'Quotes to Scrape'
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
raise an :exc:`IndexError` exception if there are no results::
raise an :exc:`IndexError` exception if there are no results:
>>> response.css('noelement')[0].get()
.. code-block:: pycon
>>> response.css("noelement")[0].get()
Traceback (most recent call last):
...
IndexError: list index out of range
You might want to use ``.get()`` directly on the
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
if there are no results::
if there are no results:
>>> response.css("noelement").get()
.. code-block:: pycon
>>> response.css("noelement").get()
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
@ -300,14 +320,16 @@ Besides the :meth:`~scrapy.selector.SelectorList.getall` and
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
:doc:`regular expressions <library/re>`:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
.. code-block:: pycon
In order to find the proper CSS selectors to use, you might find useful opening
>>> response.css("title::text").re(r"Quotes.*")
['Quotes to Scrape']
>>> response.css("title::text").re(r"Q\w+")
['Quotes']
>>> response.css("title::text").re(r"(\w+) to (\w+)")
['Quotes', 'Scrape']
In order to find the proper CSS selectors to use, you might find it useful to open
the response page from the shell in your web browser using ``view(response)``.
You can use your browser's developer tools to inspect the HTML and come up
with a selector (see :ref:`topics-developer-tools`).
@ -323,10 +345,12 @@ XPath: a brief intro
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.xpath("//title")
[<Selector query='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath("//title/text()").get()
'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@ -379,37 +403,45 @@ like this:
Let's open up scrapy shell and play a bit to find out how to extract the data
we want::
$ scrapy shell 'https://quotes.toscrape.com'
scrapy shell 'https://quotes.toscrape.com'
We get a list of selectors for the quote HTML elements with:
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
.. code-block:: pycon
>>> response.css("div.quote")
[<Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
variable, so that we can run our CSS selectors directly on a particular quote:
>>> quote = response.css("div.quote")[0]
.. code-block:: pycon
>>> quote = response.css("div.quote")[0]
Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created:
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
.. code-block:: pycon
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
Given that the tags are a list of strings, we can use the ``.getall()`` method
to get all of them:
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. code-block:: pycon
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. invisible-code-block: python
@ -418,14 +450,17 @@ to get all of them:
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
.. code-block:: pycon
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
...
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
Extracting data in our spider
-----------------------------
@ -436,7 +471,9 @@ extraction logic above into our spider.
A Scrapy spider typically generates many dictionaries containing the data
extracted from the page. To do that, we use the ``yield`` Python keyword
in the callback, as you can see below::
in the callback, as you can see below:
.. code-block:: python
import scrapy
@ -444,16 +481,16 @@ in the callback, as you can see below::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
If you run this spider, it will output the extracted data with the log::
@ -482,7 +519,7 @@ to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
scrapy crawl quotes -o quotes.jsonl
The `JSON Lines`_ format is useful because it's stream-like, you can easily
append new records to it. It doesn't have the same problem of JSON when you run
@ -531,17 +568,23 @@ This gets the anchor element, but we want the attribute ``href``. For that,
Scrapy supports a CSS extension that lets you select the attribute contents,
like this:
>>> response.css('li.next a::attr(href)').get()
'/page/2/'
.. code-block:: pycon
>>> response.css("li.next a::attr(href)").get()
'/page/2/'
There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more):
>>> response.css('li.next a').attrib['href']
'/page/2/'
.. code-block:: pycon
>>> response.css("li.next a").attrib["href"]
'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
page, extracting data from it:
.. code-block:: python
import scrapy
@ -549,18 +592,18 @@ page, extracting data from it::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
"https://quotes.toscrape.com/page/1/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
@ -592,7 +635,9 @@ A shortcut for creating Requests
--------------------------------
As a shortcut for creating Request objects you can use
:meth:`response.follow <scrapy.http.TextResponse.follow>`::
:meth:`response.follow <scrapy.http.TextResponse.follow>`:
.. code-block:: python
import scrapy
@ -600,18 +645,18 @@ As a shortcut for creating Request objects you can use
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
"https://quotes.toscrape.com/page/1/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('span small::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("span small::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
@ -619,58 +664,72 @@ Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no
need to call urljoin. Note that ``response.follow`` just returns a Request
instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
.. skip: start
for href in response.css('ul.pager a::attr(href)'):
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes:
.. code-block:: python
for href in response.css("ul.pager a::attr(href)"):
yield response.follow(href, callback=self.parse)
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
attribute automatically. So the code can be shortened further:
for a in response.css('ul.pager a'):
.. code-block:: python
for a in response.css("ul.pager a"):
yield response.follow(a, callback=self.parse)
To create multiple requests from an iterable, you can use
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead:
anchors = response.css('ul.pager a')
.. code-block:: python
anchors = response.css("ul.pager a")
yield from response.follow_all(anchors, callback=self.parse)
or, shortening it further::
or, shortening it further:
yield from response.follow_all(css='ul.pager a', callback=self.parse)
.. code-block:: python
yield from response.follow_all(css="ul.pager a", callback=self.parse)
.. skip: end
More examples and patterns
--------------------------
Here is another spider that illustrates callbacks and following links,
this time for scraping author information::
this time for scraping author information:
.. code-block:: python
import scrapy
class AuthorSpider(scrapy.Spider):
name = 'author'
name = "author"
start_urls = ['https://quotes.toscrape.com/']
start_urls = ["https://quotes.toscrape.com/"]
def parse(self, response):
author_page_links = response.css('.author + a')
author_page_links = response.css(".author + a")
yield from response.follow_all(author_page_links, self.parse_author)
pagination_links = response.css('li.next a')
pagination_links = response.css("li.next a")
yield from response.follow_all(pagination_links, self.parse)
def parse_author(self, response):
def extract_with_css(query):
return response.css(query).get(default='').strip()
return response.css(query).get(default="").strip()
yield {
'name': extract_with_css('h3.author-title::text'),
'birthdate': extract_with_css('.author-born-date::text'),
'bio': extract_with_css('.author-description::text'),
"name": extract_with_css("h3.author-title::text"),
"birthdate": extract_with_css(".author-born-date::text"),
"bio": extract_with_css(".author-description::text"),
}
This spider will start from the main page, it will follow all the links to the
@ -718,7 +777,9 @@ spider attributes by default.
In this example, the value provided for the ``tag`` argument will be available
via ``self.tag``. You can use this to make your spider fetch only quotes
with a specific tag, building the URL based on the argument::
with a specific tag, building the URL based on the argument:
.. code-block:: python
import scrapy
@ -727,20 +788,20 @@ with a specific tag, building the URL based on the argument::
name = "quotes"
def start_requests(self):
url = 'https://quotes.toscrape.com/'
tag = getattr(self, 'tag', None)
url = "https://quotes.toscrape.com/"
tag = getattr(self, "tag", None)
if tag is not None:
url = url + 'tag/' + tag
url = url + "tag/" + tag
yield scrapy.Request(url, self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
yield response.follow(next_page, self.parse)

762
docs/news.rst
View File

@ -3,6 +3,673 @@
Release notes
=============
.. _release-2.9.0:
Scrapy 2.9.0 (2023-05-08)
-------------------------
Highlights:
- Per-domain download settings.
- Compatibility with new cryptography_ and new parsel_.
- JMESPath selectors from the new parsel_.
- Bug fixes.
Deprecations
~~~~~~~~~~~~
- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to
:class:`scrapy.extensions.feedexport.FeedSlot` and the old name is
deprecated. (:issue:`5876`)
New features
~~~~~~~~~~~~
- Settings correponding to :setting:`DOWNLOAD_DELAY`,
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
:setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors
available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`)
- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed`
signals. (:issue:`5876`)
- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a
curl command from a :class:`~scrapy.Request` object. (:issue:`5892`)
- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be
:class:`pathlib.Path` instances. (:issue:`5801`)
Bug fixes
~~~~~~~~~
- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`)
- Fixed an error when using feed postprocessing with S3 storage.
(:issue:`5500`, :issue:`5581`)
- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method.
(:issue:`5811`, :issue:`5821`)
- Fixed an error when using cryptography_ 40.0.0+ and
:setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled.
(:issue:`5857`, :issue:`5858`)
- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline`
for files on Google Cloud Storage are no longer Base64-encoded.
(:issue:`5874`, :issue:`5891`)
- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed
string values for the curl ``--data-raw`` argument, which are produced by
browsers for data that includes certain symbols. (:issue:`5899`,
:issue:`5901`)
- The :command:`parse` command now also works with async generator callbacks.
(:issue:`5819`, :issue:`5824`)
- The :command:`genspider` command now properly works with HTTPS URLs.
(:issue:`3553`, :issue:`5808`)
- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
now skips certain malformed URLs instead of raising an exception.
(:issue:`5881`)
- :func:`scrapy.utils.python.get_func_args` now supports more types of
callables. (:issue:`5872`, :issue:`5885`)
- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers.
(:issue:`5914`, :issue:`5917`)
- Fixed an error breaking user handling of send failures in
:meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
Documentation
~~~~~~~~~~~~~
- Expanded contributing docs. (:issue:`5109`, :issue:`5851`)
- Added blacken-docs_ to pre-commit and reformatted the docs with it.
(:issue:`5813`, :issue:`5816`)
- Fixed a JS issue. (:issue:`5875`, :issue:`5877`)
- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`)
- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`,
:issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`)
- Tests for most of the examples in the docs are now run as a part of CI,
found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`)
- Removed usage of deprecated Python classes. (:issue:`5849`)
- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`)
- Fixed a random failure of the ``test_feedexport.test_batch_path_differ``
test. (:issue:`5855`, :issue:`5898`)
- Updated docstrings to match output produced by parsel_ 1.8.1 so that they
don't cause test failures. (:issue:`5902`, :issue:`5919`)
- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`,
:issue:`5908`)
.. _blacken-docs: https://github.com/adamchainz/blacken-docs
.. _release-2.8.0:
Scrapy 2.8.0 (2023-02-02)
-------------------------
This is a maintenance release, with minor features, bug fixes, and cleanups.
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- The ``scrapy.utils.gz.read1`` function, deprecated in Scrapy 2.0, has now
been removed. Use the :meth:`~io.BufferedIOBase.read1` method of
:class:`~gzip.GzipFile` instead.
(:issue:`5719`)
- The ``scrapy.utils.python.to_native_str`` function, deprecated in Scrapy
2.0, has now been removed. Use :func:`scrapy.utils.python.to_unicode`
instead.
(:issue:`5719`)
- The ``scrapy.utils.python.MutableChain.next`` method, deprecated in Scrapy
2.0, has now been removed. Use
:meth:`~scrapy.utils.python.MutableChain.__next__` instead.
(:issue:`5719`)
- The ``scrapy.linkextractors.FilteringLinkExtractor`` class, deprecated
in Scrapy 2.0, has now been removed. Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead.
(:issue:`5720`)
- Support for using environment variables prefixed with ``SCRAPY_`` to
override settings, deprecated in Scrapy 2.0, has now been removed.
(:issue:`5724`)
- Support for the ``noconnect`` query string argument in proxy URLs,
deprecated in Scrapy 2.0, has now been removed. We expect proxies that used
to need it to work fine without it.
(:issue:`5731`)
- The ``scrapy.utils.python.retry_on_eintr`` function, deprecated in Scrapy
2.3, has now been removed.
(:issue:`5719`)
- The ``scrapy.utils.python.WeakKeyCache`` class, deprecated in Scrapy 2.4,
has now been removed.
(:issue:`5719`)
Deprecations
~~~~~~~~~~~~
- :exc:`scrapy.pipelines.images.NoimagesDrop` is now deprecated.
(:issue:`5368`, :issue:`5489`)
- :meth:`ImagesPipeline.convert_image
<scrapy.pipelines.images.ImagesPipeline.convert_image>` must now accept a
``response_body`` parameter.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
New features
~~~~~~~~~~~~
- Applied black_ coding style to files generated with the
:command:`genspider` and :command:`startproject` commands.
(:issue:`5809`, :issue:`5814`)
.. _black: https://black.readthedocs.io/en/stable/
- :setting:`FEED_EXPORT_ENCODING` is now set to ``"utf-8"`` in the
``settings.py`` file that the :command:`startproject` command generates.
With this value, JSON exports won’t force the use of escape sequences for
non-ASCII characters.
(:issue:`5797`, :issue:`5800`)
- The :class:`~scrapy.extensions.memusage.MemoryUsage` extension now logs the
peak memory usage during checks, and the binary unit MiB is now used to
avoid confusion.
(:issue:`5717`, :issue:`5722`, :issue:`5727`)
- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set
to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from
``None``, as the latter indicates that the default spider callback
(:meth:`~scrapy.Spider.parse`) is to be used.
(:issue:`5798`)
Bug fixes
~~~~~~~~~
- Enabled unsafe legacy SSL renegotiation to fix access to some outdated
websites.
(:issue:`5491`, :issue:`5790`)
- Fixed STARTTLS-based email delivery not working with Twisted 21.2.0 and
better.
(:issue:`5386`, :issue:`5406`)
- Fixed the :meth:`finish_exporting` method of :ref:`item exporters
<topics-exporters>` not being called for empty files.
(:issue:`5537`, :issue:`5758`)
- Fixed HTTP/2 responses getting only the last value for a header when
multiple headers with the same name are received.
(:issue:`5777`)
- Fixed an exception raised by the :command:`shell` command on some cases
when :ref:`using asyncio <using-asyncio>`.
(:issue:`5740`, :issue:`5742`, :issue:`5748`, :issue:`5759`, :issue:`5760`,
:issue:`5771`)
- When using :class:`~scrapy.spiders.CrawlSpider`, callback keyword arguments
(``cb_kwargs``) added to a request in the ``process_request`` callback of a
:class:`~scrapy.spiders.Rule` will no longer be ignored.
(:issue:`5699`)
- The :ref:`images pipeline <images-pipeline>` no longer re-encodes JPEG
files.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
- Fixed the handling of transparent WebP images by the :ref:`images pipeline
<images-pipeline>`.
(:issue:`3072`, :issue:`5766`, :issue:`5767`)
- :func:`scrapy.shell.inspect_response` no longer inhibits ``SIGINT``
(Ctrl+C).
(:issue:`2918`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
with ``unique=False`` no longer filters out links that have identical URL
*and* text.
(:issue:`3798`, :issue:`3799`, :issue:`4695`, :issue:`5458`)
- :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` now
ignores URL protocols that do not support ``robots.txt`` (``data://``,
``file://``).
(:issue:`5807`)
- Silenced the ``filelock`` debug log messages introduced in Scrapy 2.6.
(:issue:`5753`, :issue:`5754`)
- Fixed the output of ``scrapy -h`` showing an unintended ``**commands**``
line.
(:issue:`5709`, :issue:`5711`, :issue:`5712`)
- Made the active project indication in the output of :ref:`commands
<topics-commands>` more clear.
(:issue:`5715`)
Documentation
~~~~~~~~~~~~~
- Documented how to :ref:`debug spiders from Visual Studio Code
<debug-vscode>`.
(:issue:`5721`)
- Documented how :setting:`DOWNLOAD_DELAY` affects per-domain concurrency.
(:issue:`5083`, :issue:`5540`)
- Improved consistency.
(:issue:`5761`)
- Fixed typos.
(:issue:`5714`, :issue:`5744`, :issue:`5764`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Applied :ref:`black coding style <coding-style>`, sorted import statements,
and introduced :ref:`pre-commit <scrapy-pre-commit>`.
(:issue:`4654`, :issue:`4658`, :issue:`5734`, :issue:`5737`, :issue:`5806`,
:issue:`5810`)
- Switched from :mod:`os.path` to :mod:`pathlib`.
(:issue:`4916`, :issue:`4497`, :issue:`5682`)
- Addressed many issues reported by Pylint.
(:issue:`5677`)
- Improved code readability.
(:issue:`5736`)
- Improved package metadata.
(:issue:`5768`)
- Removed direct invocations of ``setup.py``.
(:issue:`5774`, :issue:`5776`)
- Removed unnecessary :class:`~collections.OrderedDict` usages.
(:issue:`5795`)
- Removed unnecessary ``__str__`` definitions.
(:issue:`5150`)
- Removed obsolete code and comments.
(:issue:`5725`, :issue:`5729`, :issue:`5730`, :issue:`5732`)
- Fixed test and CI issues.
(:issue:`5749`, :issue:`5750`, :issue:`5756`, :issue:`5762`, :issue:`5765`,
:issue:`5780`, :issue:`5781`, :issue:`5782`, :issue:`5783`, :issue:`5785`,
:issue:`5786`)
.. _release-2.7.1:
Scrapy 2.7.1 (2022-11-02)
-------------------------
New features
~~~~~~~~~~~~
- Relaxed the restriction introduced in 2.6.2 so that the
``Proxy-Authorization`` header can again be set explicitly, as long as the
proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and
for as long as that proxy URL remains the same; this restores compatibility
with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`).
Bug fixes
~~~~~~~~~
- Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options
together now produces an error instead of ignoring the former option
(:issue:`5516`, :issue:`5605`).
- Replaced deprecated :mod:`asyncio` APIs that implicitly use the current
event loop with code that explicitly requests a loop from the event loop
policy (:issue:`5685`, :issue:`5689`).
- Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`,
:issue:`5589`).
- Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`).
- Improved code that checks if generators return values, so that it no longer
fails on decorated methods and partial methods (:issue:`5323`,
:issue:`5592`, :issue:`5599`, :issue:`5691`).
Documentation
~~~~~~~~~~~~~
- Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`).
- Fixed typos (:issue:`5681`, :issue:`5694`).
Quality assurance
~~~~~~~~~~~~~~~~~
- Re-enabled some erroneously disabled flake8 checks (:issue:`5688`).
- Ignored harmless deprecation warnings from :mod:`typing` in tests
(:issue:`5686`, :issue:`5697`).
- Modernized our CI configuration (:issue:`5695`, :issue:`5696`).
.. _release-2.7.0:
Scrapy 2.7.0 (2022-10-17)
-----------------------------
Highlights:
- Added Python 3.11 support, dropped Python 3.6 support
- Improved support for :ref:`asynchronous callbacks <topics-coroutines>`
- :ref:`Asyncio support <using-asyncio>` is enabled by default on new
projects
- Output names of item fields can now be arbitrary strings
- Centralized :ref:`request fingerprinting <request-fingerprints>`
configuration is now possible
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
Python 3.7 or greater is now required; support for Python 3.6 has been dropped.
Support for the upcoming Python 3.11 has been added.
The minimum required version of some dependencies has changed as well:
- lxml_: 3.5.0 → 4.3.0
- Pillow_ (:ref:`images pipeline <images-pipeline>`): 4.0.0 → 7.1.0
- zope.interface_: 5.0.0 → 5.1.0
(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`,
:issue:`5670`, :issue:`5678`)
Deprecations
~~~~~~~~~~~~
- :meth:`ImagesPipeline.thumb_path
<scrapy.pipelines.images.ImagesPipeline.thumb_path>` must now accept an
``item`` parameter (:issue:`5504`, :issue:`5508`).
- The ``scrapy.downloadermiddlewares.decompression`` module is now
deprecated (:issue:`5546`, :issue:`5547`).
New features
~~~~~~~~~~~~
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method of :ref:`spider middlewares <topics-spider-middleware>` can now be
defined as an :term:`asynchronous generator` (:issue:`4978`).
- The output of :class:`~scrapy.Request` callbacks defined as
:ref:`coroutines <topics-coroutines>` is now processed asynchronously
(:issue:`4978`).
- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous
callbacks <topics-coroutines>` (:issue:`5657`).
- New projects created with the :command:`startproject` command have
:ref:`asyncio support <using-asyncio>` enabled by default (:issue:`5590`,
:issue:`5679`).
- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a
dictionary to customize the output name of item fields, lifting the
restriction that required output names to be valid Python identifiers, e.g.
preventing them to have whitespace (:issue:`1008`, :issue:`3266`,
:issue:`3696`).
- You can now customize :ref:`request fingerprinting <request-fingerprints>`
through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of
having to change it on every Scrapy component that relies on request
fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`,
:issue:`4524`).
- ``jsonl`` is now supported and encouraged as a file extension for `JSON
Lines`_ files (:issue:`4848`).
.. _JSON Lines: https://jsonlines.org/
- :meth:`ImagesPipeline.thumb_path
<scrapy.pipelines.images.ImagesPipeline.thumb_path>` now receives the
source :ref:`item <topics-items>` (:issue:`5504`, :issue:`5508`).
Bug fixes
~~~~~~~~~
- When using Google Cloud Storage with a :ref:`media pipeline
<topics-media-pipeline>`, :setting:`FILES_EXPIRES` now also works when
:setting:`FILES_STORE` does not point at the root of your Google Cloud
Storage bucket (:issue:`5317`, :issue:`5318`).
- The :command:`parse` command now supports :ref:`asynchronous callbacks
<topics-coroutines>` (:issue:`5424`, :issue:`5577`).
- When using the :command:`parse` command with a URL for which there is no
available spider, an exception is no longer raised (:issue:`3264`,
:issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`).
- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte
order mark`_ when determining the text encoding of the response body,
following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`).
.. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
.. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
- MIME sniffing takes the response body into account in FTP and HTTP/1.0
requests, as well as in cached requests (:issue:`4873`).
- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag
is missing (:issue:`4873`).
- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value
that does not match the asyncio event loop actually installed
(:issue:`5529`).
- Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
returning only the last header (:issue:`5515`, :issue:`5526`).
- Fixed :class:`LinkExtractor
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` not ignoring the
``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`,
:issue:`4066`)
Documentation
~~~~~~~~~~~~~
- Clarified the return type of :meth:`Spider.parse <scrapy.Spider.parse>`
(:issue:`5602`, :issue:`5608`).
- To enable
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
to do `brotli compression`_, installing brotli_ is now recommended instead
of installing brotlipy_, as the former provides a more recent version of
brotli.
.. _brotli: https://github.com/google/brotli
.. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt
- :ref:`Signal documentation <topics-signals>` now mentions :ref:`coroutine
support <topics-coroutines>` and uses it in code examples (:issue:`4852`,
:issue:`5358`).
- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_
(:issue:`3582`, :issue:`5432`).
.. _Common Crawl: https://commoncrawl.org/
.. _Google cache: http://www.googleguide.com/cached_pages.html
- The new :ref:`topics-components` topic covers enforcing requirements on
Scrapy components, like :ref:`downloader middlewares
<topics-downloader-middleware>`, :ref:`extensions <topics-extensions>`,
:ref:`item pipelines <topics-item-pipeline>`, :ref:`spider middlewares
<topics-spider-middleware>`, and more; :ref:`enforce-asyncio-requirement`
has also been added (:issue:`4978`).
- :ref:`topics-settings` now indicates that setting values must be
:ref:`picklable <pickle-picklable>` (:issue:`5607`, :issue:`5629`).
- Removed outdated documentation (:issue:`5446`, :issue:`5373`,
:issue:`5369`, :issue:`5370`, :issue:`5554`).
- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`,
:issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`).
- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`,
:issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`).
Quality assurance
~~~~~~~~~~~~~~~~~
- Added a continuous integration job to run `twine check`_ (:issue:`5655`,
:issue:`5656`).
.. _twine check: https://twine.readthedocs.io/en/stable/#twine-check
- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`,
:issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`,
:issue:`5671`, :issue:`5675`).
- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`,
:issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`).
- Applied minor code improvements (:issue:`5661`).
.. _release-2.6.3:
Scrapy 2.6.3 (2022-09-27)
-------------------------
- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3
(:issue:`5634`, :issue:`5635`, :issue:`5636`).
- Upgraded the minimum versions of the following dependencies:
- cryptography_: 2.0 → 3.3
- pyOpenSSL_: 16.2.0 → 21.0.0
- service_identity_: 16.0.0 → 18.1.0
- Twisted_: 17.9.0 → 18.9.0
- zope.interface_: 4.1.3 → 5.0.0
(:issue:`5621`, :issue:`5632`)
- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`,
:issue:`5631`).
.. _release-2.6.2:
Scrapy 2.6.2 (2022-07-25)
-------------------------
**Security bug fix:**
- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
processes a request with :reqmeta:`proxy` metadata, and that
:reqmeta:`proxy` metadata includes proxy credentials,
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
the ``Proxy-Authorization`` header, but only if that header is not already
set.
There are third-party proxy-rotation downloader middlewares that set
different :reqmeta:`proxy` metadata every time they process a request.
Because of request retries and redirects, the same request can be processed
by downloader middlewares more than once, including both
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
any third-party proxy-rotation downloader middleware.
These third-party proxy-rotation downloader middlewares could change the
:reqmeta:`proxy` metadata of a request to a new value, but fail to remove
the ``Proxy-Authorization`` header from the previous value of the
:reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
to a different proxy.
To prevent the unintended leaking of proxy credentials, the behavior of
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
as follows when processing a request:
- If the request being processed defines :reqmeta:`proxy` metadata that
includes credentials, the ``Proxy-Authorization`` header is always
updated to feature those credentials.
- If the request being processed defines :reqmeta:`proxy` metadata
without credentials, the ``Proxy-Authorization`` header is removed
*unless* it was originally defined for the same proxy URL.
To remove proxy credentials while keeping the same proxy URL, remove
the ``Proxy-Authorization`` header.
- If the request has no :reqmeta:`proxy` metadata, or that metadata is a
falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
removed.
It is no longer possible to set a proxy URL through the
:reqmeta:`proxy` metadata but set the credentials through the
``Proxy-Authorization`` header. Set proxy credentials through the
:reqmeta:`proxy` metadata instead.
Also fixes the following regressions introduced in 2.6.0:
- :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
spiders (:issue:`5435`, :issue:`5436`)
- Installing a Twisted reactor before Scrapy does (e.g. importing
:mod:`twisted.internet.reactor` somewhere at the module level) no longer
prevents Scrapy from starting, as long as a different reactor is not
specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`)
- Fixed an exception that was being logged after the spider finished under
certain conditions (:issue:`5437`, :issue:`5440`)
- The ``--output``/``-o`` command-line parameter supports again a value
starting with a hyphen (:issue:`5444`, :issue:`5445`)
- The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`,
:issue:`5482`)
.. _release-2.6.1:
Scrapy 2.6.1 (2022-03-01)
@ -113,6 +780,9 @@ Backward-incompatible changes
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
other exceptions would be raised at run time. (:issue:`3559`)
- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects
is now named ``encoding``, in line with all other fields (:issue:`5130`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
@ -1643,7 +2313,7 @@ New features
:issue:`4370`)
* A new ``keep_fragments`` parameter of
:func:`scrapy.utils.request.request_fingerprint` allows to generate
``scrapy.utils.request.request_fingerprint`` allows to generate
different fingerprints for requests with different fragments in their URL
(:issue:`4104`)
@ -1897,6 +2567,59 @@ affect subclasses:
(:issue:`3884`)
.. _release-1.8.3:
Scrapy 1.8.3 (2022-07-25)
-------------------------
**Security bug fix:**
- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
processes a request with :reqmeta:`proxy` metadata, and that
:reqmeta:`proxy` metadata includes proxy credentials,
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
the ``Proxy-Authorization`` header, but only if that header is not already
set.
There are third-party proxy-rotation downloader middlewares that set
different :reqmeta:`proxy` metadata every time they process a request.
Because of request retries and redirects, the same request can be processed
by downloader middlewares more than once, including both
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
any third-party proxy-rotation downloader middleware.
These third-party proxy-rotation downloader middlewares could change the
:reqmeta:`proxy` metadata of a request to a new value, but fail to remove
the ``Proxy-Authorization`` header from the previous value of the
:reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
to a different proxy.
To prevent the unintended leaking of proxy credentials, the behavior of
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
as follows when processing a request:
- If the request being processed defines :reqmeta:`proxy` metadata that
includes credentials, the ``Proxy-Authorization`` header is always
updated to feature those credentials.
- If the request being processed defines :reqmeta:`proxy` metadata
without credentials, the ``Proxy-Authorization`` header is removed
*unless* it was originally defined for the same proxy URL.
To remove proxy credentials while keeping the same proxy URL, remove
the ``Proxy-Authorization`` header.
- If the request has no :reqmeta:`proxy` metadata, or that metadata is a
falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
removed.
It is no longer possible to set a proxy URL through the
:reqmeta:`proxy` metadata but set the credentials through the
``Proxy-Authorization`` header. Set proxy credentials through the
:reqmeta:`proxy` metadata instead.
.. _release-1.8.2:
Scrapy 1.8.2 (2022-03-01)
@ -2022,11 +2745,13 @@ Backward-incompatible changes
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
into lists:
>>> item = MyItem()
>>> item['field'] = 'value1'
>>> loader = ItemLoader(item=item)
>>> item['field']
['value1']
.. code-block:: pycon
>>> item = MyItem()
>>> item["field"] = "value1"
>>> loader = ItemLoader(item=item)
>>> item["field"]
['value1']
This is needed to allow adding values to existing fields
(``loader.add_value('field', 'value2')``).
@ -2985,7 +3710,7 @@ New Features
~~~~~~~~~~~~
- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
- Support `brotli`_-compressed content; requires optional `brotlipy`_
- Support `brotli-compressed`_ content; requires optional `brotlipy`_
(:issue:`2535`)
- New :ref:`response.follow <response-follow-example>` shortcut
for creating requests (:issue:`1940`)
@ -3022,7 +3747,7 @@ New Features
- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
(:issue:`2740`)
.. _brotli: https://github.com/google/brotli
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://github.com/python-hyper/brotlipy/
Bug fixes
@ -3604,8 +4329,6 @@ Relocations
+ Note: telnet is not enabled on Python 3
(https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
.. _parsel: https://github.com/scrapy/parsel
Bugfixes
~~~~~~~~
@ -4305,7 +5028,7 @@ Scrapy 0.22.1 (released 2014-02-08)
- BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
- BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
- Fix tests for Travis-CI build (:commit:`76c7e20`)
- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
- Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
- Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
@ -4330,7 +5053,7 @@ Enhancements
- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
- Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`)
- Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
- Selectors register EXSLT namespaces by default (:issue:`472`)
- Unify item loaders similar to selectors renaming (:issue:`461`)
@ -4510,7 +5233,7 @@ Scrapy 0.18.0 (released 2013-08-09)
-----------------------------------
- Lot of improvements to testsuite run using Tox, including a way to test on pypi
- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`)
- Use lxml recover option to parse sitemaps (:issue:`347`)
- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
@ -4544,8 +5267,8 @@ Scrapy 0.18.0 (released 2013-08-09)
- Added ``--pdb`` option to ``scrapy`` command line tool
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
- Several improvements to spider contracts
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections,
- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62
- added from_crawler method to spiders
- added system tests with mock server
- more improvements to macOS compatibility (thanks Alex Cepoi)
@ -4687,7 +5410,7 @@ Scrapy changes:
- promoted :ref:`topics-djangoitem` to main contrib
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- removed signal: ``scrapy.mail.mail_sent``
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
- DBM is now the default storage backend for HTTP cache middleware
@ -4753,7 +5476,7 @@ Scrapy 0.14
New features and settings
~~~~~~~~~~~~~~~~~~~~~~~~~
- Support for `AJAX crawleable urls`_
- Support for `AJAX crawlable urls`_
- New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
- added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
- Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
@ -5013,7 +5736,7 @@ Backward-incompatible changes
- Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
- Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
- Refactored HTTP Cache middleware
- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
- Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
- Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
- Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
@ -5024,7 +5747,7 @@ Scrapy 0.7
First release of Scrapy.
.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
@ -5035,6 +5758,7 @@ First release of Scrapy.
.. _LevelDB: https://github.com/google/leveldb
.. _lxml: https://lxml.de/
.. _marshal: https://docs.python.org/2/library/marshal.html
.. _parsel: https://github.com/scrapy/parsel
.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx-rtd-theme>=0.5.2
sphinx==5.0.2
sphinx-hoverxref==1.1.1
sphinx-notfound-page==0.8
sphinx-rtd-theme==1.0.0

21
docs/topics/api.rst
View File

@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
.. attribute:: request_fingerprinter
The request fingerprint builder of this crawler.
This is used from extensions and middlewares to build short, unique
identifiers for requests. See :ref:`request-fingerprints`.
.. attribute:: settings
The settings manager of this crawler.
@ -125,16 +132,14 @@ Settings API
precedence over lesser ones when setting and retrieving values in the
:class:`~scrapy.settings.Settings` class.
.. highlight:: python
::
.. code-block:: python
SETTINGS_PRIORITIES = {
'default': 0,
'command': 10,
'project': 20,
'spider': 30,
'cmdline': 40,
"default": 0,
"command": 10,
"project": 20,
"spider": 30,
"cmdline": 40,
}
For a detailed explanation on each settings sources, see:

26
docs/topics/asyncio.rst
View File

@ -96,3 +96,29 @@ Futures. Scrapy provides two helpers for this:
down to Scrapy 2.0 (earlier versions do not support
:mod:`asyncio`), you can copy the implementation of these functions
into your own code.
.. _enforce-asyncio-requirement:
Enforcing asyncio as a requirement
==================================
If you are writing a :ref:`component <topics-components>` that requires asyncio
to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
:ref:`enforce it as a requirement <enforce-component-requirements>`. For
example:
.. code-block:: python
from scrapy.utils.reactor import is_asyncio_reactor_installed
class MyComponent:
def __init__(self):
if not is_asyncio_reactor_installed():
raise ValueError(
f"{MyComponent.__qualname__} requires the asyncio Twisted "
f"reactor. Make sure you have it configured in the "
f"TWISTED_REACTOR setting. See the asyncio documentation "
f"of Scrapy for more information."
)

42
docs/topics/broad-crawls.rst
View File

@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ
It works best during single-domain crawl. It does not work well with crawling
many different domains in parallel
To apply the recommended priority queue use::
To apply the recommended priority queue use:
SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
.. code-block:: python
SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"
.. _broad-crawls-concurrency:
@ -68,10 +70,12 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
to increase it will depend on how much CPU and memory you crawler will have
to increase it will depend on how much CPU and memory your crawler will have
available.
A good starting point is ``100``::
A good starting point is ``100``:
.. code-block:: python
CONCURRENT_REQUESTS = 100
@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of
threads handling DNS queries. The DNS queue will be processed faster speeding
up establishing of connection and crawling overall.
To increase maximum thread pool size use::
To increase maximum thread pool size use:
.. code-block:: python
REACTOR_THREADPOOL_MAXSIZE = 20
@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in
production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::
To set the log level use:
LOG_LEVEL = 'INFO'
.. code-block:: python
LOG_LEVEL = "INFO"
Disable cookies
===============
@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve
performance by saving some CPU cycles and reducing the memory footprint of your
Scrapy crawler.
To disable cookies use::
To disable cookies use:
.. code-block:: python
COOKIES_ENABLED = False
@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout
error which gets retried many times, unnecessarily, preventing crawler capacity
to be reused for other domains.
To disable retries use::
To disable retries use:
.. code-block:: python
RETRY_ENABLED = False
@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the
case for broad crawls) reduce the download timeout so that stuck requests are
discarded quickly and free up capacity to process the next ones.
To reduce the download timeout use::
To reduce the download timeout use:
.. code-block:: python
DOWNLOAD_TIMEOUT = 15
@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of
request constant per crawl batch, otherwise redirect loops may cause the
crawler to dedicate too many resources on any specific domain.
To disable redirects use::
To disable redirects use:
.. code-block:: python
REDIRECT_ENABLED = False
@ -179,7 +195,9 @@ Pages can indicate it in two ways:
"main", "index" website pages.
Scrapy handles (1) automatically; to handle (2) enable
:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`::
:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`:
.. code-block:: python
AJAXCRAWL_ENABLED = True

40
docs/topics/commands.rst
View File

@ -238,9 +238,6 @@ genspider
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. note:: Even if an HTTPS URL is specified, the protocol used in
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
Usage example::
$ scrapy genspider -l
@ -271,11 +268,31 @@ crawl
Start crawling using a spider.
Supported options:
* ``-h, --help``: show a help message and exit
* ``-a NAME=VALUE``: set a spider argument (may be repeated)
* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
Usage examples::
$ scrapy crawl myspider
[ ... myspider starts crawling ... ]
$ scrapy crawl -o myfile:csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
$ scrapy crawl -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
$ scrapy crawl -o myfile -t csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check
@ -597,7 +614,7 @@ Example:
.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
COMMANDS_MODULE = "mybot.commands"
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
@ -616,10 +633,11 @@ The following example adds ``my_command`` command:
from setuptools import setup, find_packages
setup(name='scrapy-mymodule',
entry_points={
'scrapy.commands': [
'my_command=my_scrapy_module.commands:MyCommand',
],
},
)
setup(
name="scrapy-mymodule",
entry_points={
"scrapy.commands": [
"my_command=my_scrapy_module.commands:MyCommand",
],
},
)

86
docs/topics/components.rst Normal file
View File

@ -0,0 +1,86 @@
.. _topics-components:
==========
Components
==========
A Scrapy component is any class whose objects are created using
:func:`scrapy.utils.misc.create_instance`.
That includes the classes that you may assign to the following settings:
- :setting:`DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings <topics-settings>`, to
modify their behavior.
.. _enforce-component-requirements:
Enforcing component requirements
================================
Sometimes, your components may only be intended to work under certain
conditions. For example, they may require a minimum version of Scrapy to work as
intended, or they may require certain settings to have specific values.
In addition to describing those conditions in the documentation of your
component, it is a good practice to raise an exception from the ``__init__``
method of your component if those conditions are not met at run time.
In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
:ref:`extensions <topics-extensions>`, :ref:`item pipelines
<topics-item-pipeline>`, and :ref:`spider middlewares
<topics-spider-middleware>`, you should raise
:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
parameter to the exception so that it is printed in the logs, for the user to
see. For other components, feel free to raise whatever other exception feels
right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
version mismatch, while :exc:`ValueError` may be better if the issue is the
value of a setting.
If your requirement is a minimum Scrapy version, you may use
:attr:`scrapy.__version__` to enforce your requirement. For example:
.. code-block:: python
from packaging.version import parse as parse_version
import scrapy
class MyComponent:
def __init__(self):
if parse_version(scrapy.__version__) < parse_version("2.7"):
raise RuntimeError(
f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
f"later, which allow defining the process_spider_output "
f"method of spider middlewares as an asynchronous "
f"generator."
)

41
docs/topics/contracts.rst
View File

@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts.
This allows you to test each callback of your spider by hardcoding a sample url
and check various constraints for how the callback processes the response. Each
contract is prefixed with an ``@`` and included in the docstring. See the
following example::
following example:
.. code-block:: python
def parse(self, response):
""" This function parses a sample response. Some contracts are mingled
"""
This function parses a sample response. Some contracts are mingled
with this docstring.
@url http://www.amazon.com/s?field-keywords=selfish+gene
@ -64,11 +67,13 @@ Custom Contracts
If you find you need more power than the built-in Scrapy contracts you can
create and load your own contracts in the project by using the
:setting:`SPIDER_CONTRACTS` setting::
:setting:`SPIDER_CONTRACTS` setting:
.. code-block:: python
SPIDER_CONTRACTS = {
'myproject.contracts.ResponseCheck': 10,
'myproject.contracts.ItemValidate': 10,
"myproject.contracts.ResponseCheck": 10,
"myproject.contracts.ItemValidate": 10,
}
Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
@ -102,7 +107,7 @@ override three methods:
.. method:: Contract.post_process(output)
This allows processing the output of the callback. Iterators are
converted listified before being passed to this hook.
converted to lists before being passed to this hook.
Raise :class:`~scrapy.exceptions.ContractFail` from
:class:`~scrapy.contracts.Contract.pre_process` or
@ -111,22 +116,27 @@ Raise :class:`~scrapy.exceptions.ContractFail` from
.. autoclass:: scrapy.exceptions.ContractFail
Here is a demo contract which checks the presence of a custom header in the
response received::
response received:
.. skip: next
.. code-block:: python
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
class HasHeaderContract(Contract):
""" Demo contract which checks the presence of a custom header
@has_header X-CustomHeader
"""
Demo contract which checks the presence of a custom header
@has_header X-CustomHeader
"""
name = 'has_header'
name = "has_header"
def pre_process(self, response):
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
raise ContractFail("X-CustomHeader not present")
.. _detecting-contract-check-runs:
@ -135,14 +145,17 @@ Detecting check runs
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
your spiders or your settings when ``scrapy check`` is used::
your spiders or your settings when ``scrapy check`` is used:
.. code-block:: python
import os
import scrapy
class ExampleSpider(scrapy.Spider):
name = 'example'
name = "example"
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
if os.environ.get("SCRAPY_CHECK"):
pass # Do some scraper adjustments when a check is running

218
docs/topics/coroutines.rst
View File

@ -1,3 +1,5 @@
.. _topics-coroutines:
==========
Coroutines
==========
@ -17,14 +19,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.Request` callbacks.
.. note:: The callback output is not processed until the whole callback
finishes.
If you are using any custom or third-party :ref:`spider middleware
<topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
As a side effect, if the callback raises an exception, none of its
output is processed.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
.. versionchanged:: 2.7
Output of async callbacks is now processed asynchronously instead of
collecting all of it first.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -39,54 +39,78 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
Usage
=====
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method of :ref:`spider middlewares <topics-spider-middleware>`.
There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
It must be defined as an :term:`asynchronous generator`. The input
``result`` parameter is an :term:`asynchronous iterable`.
See also :ref:`sync-async-spider-middleware` and
:ref:`universal-spider-middleware`.
.. versionadded:: 2.7
General usage
=============
There are several use cases for coroutines in Scrapy.
Code that would return Deferreds when written for previous Scrapy versions,
such as downloader middlewares and signal handlers, can be rewritten to be
shorter and cleaner:
.. code-block:: python
from itemadapter import ItemAdapter
class DbPipeline:
def _update_item(self, data, item):
adapter = ItemAdapter(item)
adapter['field'] = data
adapter["field"] = data
return item
def process_item(self, item, spider):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter['id'])
dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
becomes:
.. code-block:: python
from itemadapter import ItemAdapter
class DbPipeline:
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
adapter['field'] = await db.get_some_data(adapter['id'])
adapter["field"] = await db.get_some_data(adapter["id"])
return item
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
This means you can use many useful Python libraries providing such code:
.. skip: next
.. code-block:: python
class MySpiderDeferred(Spider):
# ...
async def parse(self, response):
additional_response = await treq.get('https://additional.url')
additional_response = await treq.get("https://additional.url")
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
class MySpiderAsyncio(Spider):
# ...
async def parse(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
async with session.get("https://additional.url") as additional_response:
additional_data = await additional_response.text()
# ... use response and additional_data to yield items and requests
@ -104,7 +128,159 @@ Common use cases for asynchronous code include:
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
(see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs
.. _inline-requests:
Inline requests
===============
The spider below shows how to send a request and await its response all from
within a spider callback:
.. code-block:: python
from scrapy import Spider, Request
from scrapy.utils.defer import maybe_deferred_to_future
class SingleRequestSpider(Spider):
name = "single"
start_urls = ["https://example.org/product"]
async def parse(self, response, **kwargs):
additional_request = Request("https://example.org/price")
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
yield {
"h1": response.css("h1").get(),
"price": additional_response.css("#price").get(),
}
You can also send multiple requests in parallel:
.. code-block:: python
from scrapy import Spider, Request
from scrapy.utils.defer import maybe_deferred_to_future
from twisted.internet.defer import DeferredList
class MultipleRequestsSpider(Spider):
name = "multiple"
start_urls = ["https://example.com/product"]
async def parse(self, response, **kwargs):
additional_requests = [
Request("https://example.com/price"),
Request("https://example.com/color"),
]
deferreds = []
for r in additional_requests:
deferred = self.crawler.engine.download(r)
deferreds.append(deferred)
responses = await maybe_deferred_to_future(DeferredList(deferreds))
yield {
"h1": response.css("h1::text").get(),
"price": responses[0][1].css(".price::text").get(),
"price2": responses[1][1].css(".color::text").get(),
}
.. _sync-async-spider-middleware:
Mixing synchronous and asynchronous spider middlewares
======================================================
.. versionadded:: 2.7
The output of a :class:`~scrapy.Request` callback is passed as the ``result``
parameter to the
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
of the first :ref:`spider middleware <topics-spider-middleware>` from the
:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
Then the output of that ``process_spider_output`` method is passed to the
``process_spider_output`` method of the next spider middleware, and so on for
every active spider middleware.
Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
in this chain of calls.
However, if any of the ``process_spider_output`` methods is defined as a
synchronous method, and the previous ``Request`` callback or
``process_spider_output`` method is a coroutine, there are some drawbacks to
the asynchronous-to-synchronous conversion that Scrapy does so that the
synchronous ``process_spider_output`` method gets a synchronous iterable as its
``result`` parameter:
- The whole output of the previous ``Request`` callback or
``process_spider_output`` method is awaited at this point.
- If an exception raises while awaiting the output of the previous
``Request`` callback or ``process_spider_output`` method, none of that
output will be processed.
This contrasts with the regular behavior, where all items yielded before
an exception raises are processed.
Asynchronous-to-synchronous conversions are supported for backward
compatibility, but they are deprecated and will stop working in a future
version of Scrapy.
To avoid asynchronous-to-synchronous conversions, when defining ``Request``
callbacks as coroutine methods or when using spider middlewares whose
``process_spider_output`` method is an :term:`asynchronous generator`, all
active spider middlewares must either have their ``process_spider_output``
method defined as an asynchronous generator or :ref:`define a
process_spider_output_async method <universal-spider-middleware>`.
.. note:: When using third-party spider middlewares that only define a
synchronous ``process_spider_output`` method, consider
:ref:`making them universal <universal-spider-middleware>` through
:ref:`subclassing <tut-inheritance>`.
.. _universal-spider-middleware:
Universal spider middlewares
============================
.. versionadded:: 2.7
To allow writing a spider middleware that supports asynchronous execution of
its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
while maintaining support for older Scrapy versions, you may define
``process_spider_output`` as a synchronous method and define an
:term:`asynchronous generator` version of that method with an alternative name:
``process_spider_output_async``.
For example:
.. code-block:: python
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
# ... do something with r
yield r
async def process_spider_output_async(self, response, result, spider):
async for r in result:
# ... do something with r
yield r
.. note:: This is an interim measure to allow, for a time, to write code that
works in Scrapy 2.7 and later without requiring
asynchronous-to-synchronous conversions, and works in earlier Scrapy
versions as well.
In some future version of Scrapy, however, this feature will be
deprecated and, eventually, in a later version of Scrapy, this
feature will be removed, and all spider middlewares will be expected
to define their ``process_spider_output`` method as an asynchronous
generator.

66
docs/topics/debug.rst
View File

@ -5,21 +5,25 @@ Debugging Spiders
=================
This document explains the most common techniques for debugging spiders.
Consider the following Scrapy spider below::
Consider the following Scrapy spider below:
.. skip: next
.. code-block:: python
import scrapy
from myproject.items import MyItem
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
start_urls = (
'http://example.com/page1',
'http://example.com/page2',
)
"http://example.com/page1",
"http://example.com/page2",
)
def parse(self, response):
# <processing code not shown>
# collect `item_urls`
# collect `item_urls`
for item_url in item_urls:
yield scrapy.Request(item_url, self.parse_item)
@ -28,7 +32,9 @@ Consider the following Scrapy spider below::
item = MyItem()
# populate `item` fields
# and extract item_details_url
yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
yield scrapy.Request(
item_details_url, self.parse_details, cb_kwargs={"item": item}
)
def parse_details(self, response, item):
# populate more `item` fields
@ -103,10 +109,13 @@ showing the response received and the output. How to debug the situation when
.. highlight:: python
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
:ref:`topics-shell-inspect-response`):
.. code-block:: python
from scrapy.shell import inspect_response
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
@ -121,10 +130,13 @@ Open in browser
Sometimes you just want to see how a certain response looks in a browser, you
can use the ``open_in_browser`` function for that. Here is an example of how
you would use it::
you would use it:
.. code-block:: python
from scrapy.utils.response import open_in_browser
def parse_details(self, response):
if "item name" not in response.body:
open_in_browser(response)
@ -138,15 +150,47 @@ Logging
Logging is another useful option for getting information about your spider run.
Although not as convenient, it comes with the advantage that the logs will be
available in all future runs should they be necessary again::
available in all future runs should they be necessary again:
.. code-block:: python
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
else:
self.logger.warning('No item received for %s', response.url)
self.logger.warning("No item received for %s", response.url)
For more information, check the :ref:`topics-logging` section.
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
.. _debug-vscode:
Visual Studio Code
==================
.. highlight:: json
To debug spiders with Visual Studio Code you can use the following ``launch.json``::
{
"version": "0.1.0",
"configurations": [
{
"name": "Python: Launch Scrapy Spider",
"type": "python",
"request": "launch",
"module": "scrapy",
"args": [
"runspider",
"${file}"
],
"console": "integratedTerminal"
}
]
}
Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in
your Scrapy spider.

35
docs/topics/developer-tools.rst
View File

@ -94,8 +94,10 @@ Then, back to your web browser, right-click on the ``span`` tag, select
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
.. code-block:: pycon
>>> response.xpath("/html/body/div/div[2]/div[1]/div[1]/span[1]/text()").getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@ -124,11 +126,13 @@ With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
.. code-block:: pycon
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@ -237,17 +241,19 @@ on the request and open ``Open in new tab`` to get a better overview.
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
also request each page to get every quote on the site:
.. code-block:: python
import scrapy
import json
class QuoteSpider(scrapy.Spider):
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
name = "quote"
allowed_domains = ["quotes.toscrape.com"]
page = 1
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
def parse(self, response):
data = json.loads(response.text)
@ -275,7 +281,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
format, by right-clicking on each of them in the network tool and using the
:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
request::
request:
.. code-block:: python
from scrapy import Request
@ -286,7 +294,8 @@ request::
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
"zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
"I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'"
)
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`

88
docs/topics/downloader-middleware.rst
View File

@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the
:setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the
middleware class paths and their values are the middleware orders.
Here's an example::
Here's an example:
.. code-block:: python
DOWNLOADER_MIDDLEWARES = {
'myproject.middlewares.CustomDownloaderMiddleware': 543,
"myproject.middlewares.CustomDownloaderMiddleware": 543,
}
The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied.
If you want to disable a built-in middleware (the ones defined in
:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None``
as its value. For example, if you want to disable the user-agent middleware::
as its value. For example, if you want to disable the user-agent middleware:
.. code-block:: python
DOWNLOADER_MIDDLEWARES = {
'myproject.middlewares.CustomDownloaderMiddleware': 543,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
"myproject.middlewares.CustomDownloaderMiddleware": 543,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -226,20 +230,26 @@ There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
For example::
For example:
.. skip: next
.. code-block:: python
for i, url in enumerate(urls):
yield scrapy.Request(url, meta={'cookiejar': i},
callback=self.parse_page)
yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page)
Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep
passing it along on subsequent requests. For example::
passing it along on subsequent requests. For example:
.. code-block:: python
def parse_page(self, response):
# do some processing
return scrapy.Request("http://www.example.com/otherpage",
meta={'cookiejar': response.meta['cookiejar']},
callback=self.parse_other_page)
return scrapy.Request(
"http://www.example.com/otherpage",
meta={"cookiejar": response.meta["cookiejar"]},
callback=self.parse_other_page,
)
.. setting:: COOKIES_ENABLED
@ -339,16 +349,18 @@ HttpAuthMiddleware
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example::
Example:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class SomeIntranetSiteSpider(CrawlSpider):
http_user = 'someuser'
http_pass = 'somepass'
http_auth_domain = 'intranet.example.com'
name = 'intranet.example.com'
class SomeIntranetSiteSpider(CrawlSpider):
http_user = "someuser"
http_pass = "somepass"
http_auth_domain = "intranet.example.com"
name = "intranet.example.com"
# .. rest of the spider code omitted ...
@ -792,7 +804,9 @@ If you want to handle some redirect status codes in your spider, you can
specify these in the ``handle_httpstatus_list`` spider attribute.
For example, if you want the redirect middleware to ignore 301 and 302
responses (and pass them through to your spider) you can do this::
responses (and pass them through to your spider) you can do this:
.. code-block:: python
class MySpider(CrawlSpider):
handle_httpstatus_list = [301, 302]
@ -901,6 +915,7 @@ settings (see the settings documentation for more info):
* :setting:`RETRY_ENABLED`
* :setting:`RETRY_TIMES`
* :setting:`RETRY_HTTP_CODES`
* :setting:`RETRY_EXCEPTIONS`
.. reqmeta:: dont_retry
@ -952,10 +967,41 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_EXCEPTIONS
RETRY_EXCEPTIONS
^^^^^^^^^^^^^^^^
Default::
[
'twisted.internet.defer.TimeoutError',
'twisted.internet.error.TimeoutError',
'twisted.internet.error.DNSLookupError',
'twisted.internet.error.ConnectionRefusedError',
'twisted.internet.error.ConnectionDone',
'twisted.internet.error.ConnectError',
'twisted.internet.error.ConnectionLost',
'twisted.internet.error.TCPTimedOutError',
'twisted.web.client.ResponseFailed',
IOError,
'scrapy.core.downloader.handlers.http11.TunnelError',
]
List of exceptions to retry.
Each list entry may be an exception type or its import path as a string.
An exception will not be caught when the exception type is not in
:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
exception propagation, see
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
^^^^^^^^^^^^^^^^^^^^^
Default: ``-1``
@ -1119,7 +1165,7 @@ In order to use this parser:
.. _support-for-new-robots-parser:
Implementing support for a new parser
-------------------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can implement support for a new robots.txt_ parser by subclassing
the abstract base class :class:`~scrapy.robotstxt.RobotParser` and

57
docs/topics/dynamic-content.rst
View File

@ -119,16 +119,20 @@ data from it depends on the type of response:
<topics-selectors>` as usual.
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
:attr:`response.text <scrapy.http.TextResponse.text>`:
.. code-block:: python
data = json.loads(response.text)
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a
:class:`~scrapy.Selector` and then
:ref:`use it <topics-selectors>` as usual::
:ref:`use it <topics-selectors>` as usual:
selector = Selector(data['html'])
.. code-block:: python
selector = Selector(data["html"])
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
@ -179,10 +183,12 @@ data from it:
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
>>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
>>> json_data = response.css('script::text').re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
.. code-block:: pycon
>>> pattern = r"\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n"
>>> json_data = response.css("script::text").re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
@ -190,11 +196,13 @@ data from it:
``var data = {field: "value", secondField: "second value"};``
you can extract that data as follows:
>>> import chompjs
>>> javascript = response.css('script::text').get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
.. code-block:: pycon
>>> import chompjs
>>> javascript = response.css("script::text").get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
@ -202,14 +210,16 @@ data from it:
For example, if the JavaScript code contains
``var data = {field: "value"};`` you can extract that data as follows:
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css('script::text').get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. code-block:: pycon
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css("script::text").get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding="unicode")
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. _topics-javascript-rendering:
@ -250,11 +260,14 @@ automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
One such library is `playwright-python`_ (an official Python port of `playwright`_).
The following is a simple snippet to illustrate its usage within a Scrapy spider::
The following is a simple snippet to illustrate its usage within a Scrapy spider:
.. code-block:: python
import scrapy
from playwright.async_api import async_playwright
class PlaywrightSpider(scrapy.Spider):
name = "playwright"
start_urls = ["data:,"] # avoid using the default Scrapy downloader
@ -263,7 +276,7 @@ The following is a simple snippet to illustrate its usage within a Scrapy spider
async with async_playwright() as pw:
browser = await pw.chromium.launch()
page = await browser.new_page()
await page.goto("https:/example.org")
await page.goto("https://example.org")
title = await page.title()
return {"title": title}

22
docs/topics/email.rst
View File

@ -19,19 +19,33 @@ Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
the standard ``__init__`` method::
the standard ``__init__`` method:
.. code-block:: python
from scrapy.mail import MailSender
mailer = MailSender()
Or you can instantiate it passing a Scrapy settings object, which will respect
the :ref:`settings <topics-email-settings>`::
the :ref:`settings <topics-email-settings>`:
.. skip: start
.. code-block:: python
mailer = MailSender.from_settings(settings)
And here is how to use it to send an e-mail (without attachments)::
And here is how to use it to send an e-mail (without attachments):
mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
.. code-block:: python
mailer.send(
to=["someone@example.com"],
subject="Some subject",
body="Some body",
cc=["another@example.com"],
)
.. skip: end
MailSender class reference
==========================

8
docs/topics/exceptions.rst
View File

@ -26,11 +26,13 @@ CloseSpider
:param reason: the reason for closing
:type reason: str
For example::
For example:
.. code-block:: python
def parse_page(self, response):
if 'Bandwidth exceeded' in response.body:
raise CloseSpider('bandwidth_exceeded')
if "Bandwidth exceeded" in response.body:
raise CloseSpider("bandwidth_exceeded")
DontCloseSpider
---------------

68
docs/topics/exporters.rst
View File

@ -38,11 +38,14 @@ the end of the exporting process
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
value of one of their fields:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
@ -56,9 +59,9 @@ value of one of their fields::
def _exporter_for_item(self, item):
adapter = ItemAdapter(item)
year = adapter['year']
year = adapter["year"]
if year not in self.year_to_exporter:
xml_file = open(f'{year}.xml', 'wb')
xml_file = open(f"{year}.xml", "wb")
exporter = XmlItemExporter(xml_file)
exporter.start_exporting()
self.year_to_exporter[year] = (exporter, xml_file)
@ -94,12 +97,16 @@ If you use :class:`~scrapy.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
Example:
.. code-block:: python
import scrapy
def serialize_price(value):
return f'$ {str(value)}'
return f"$ {str(value)}"
class Product(scrapy.Item):
name = scrapy.Field()
@ -115,15 +122,17 @@ customize how your field value will be exported.
Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method
after your custom code.
Example::
Example:
.. code-block:: python
from scrapy.exporters import XmlItemExporter
from scrapy.exporter import XmlItemExporter
class ProductXmlExporter(XmlItemExporter):
def serialize_field(self, field, name, value):
if name == 'price':
return f'$ {str(value)}'
if name == "price":
return f"$ {str(value)}"
return super().serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -132,10 +141,13 @@ Built-in Item Exporters reference
=================================
Here is a list of the Item Exporters bundled with Scrapy. Some of them contain
output examples, which assume you're exporting these two items::
output examples, which assume you're exporting these two items:
Item(name='Color TV', price='1200')
Item(name='DVD player', price='200')
.. skip: next
.. code-block:: python
Item(name="Color TV", price="1200")
Item(name="DVD player", price="200")
BaseItemExporter
----------------
@ -195,17 +207,25 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or ``None`` if
you want to export all fields. Defaults to ``None``.
Fields to export, their order [1]_ and their output names.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Possible values are:
When using :ref:`item objects <item-types>` that do not expose all their
possible fields, exporters that do not support exporting a different
subset of fields per item will only export the fields found in the first
item exported. Use ``fields_to_export`` to define all the fields to be
exported.
- ``None`` (all fields [2]_, default)
- A list of fields::
['field1', 'field2']
- A dict where keys are fields and values are output names::
{'field1': 'Field 1', 'field2': 'Field 2'}
.. [1] Not all exporters respect the specified field order.
.. [2] When using :ref:`item objects <item-types>` that do not expose
all their possible fields, exporters that do not support exporting
a different subset of fields per item will only export the fields
found in the first item exported.
.. attribute:: export_empty_fields
@ -297,8 +317,8 @@ CsvItemExporter
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
CSV columns, their order and their column names. The
:attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)

26
docs/topics/extensions.rst
View File

@ -17,7 +17,7 @@ settings, just like any other Scrapy code.
It is customary for extensions to prefix their settings with their own name, to
avoid collision with existing (and future) extensions. For example, a
hypothetic extension to handle `Google Sitemaps`_ would use settings like
hypothetical extension to handle `Google Sitemaps`_ would use settings like
``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
@ -31,11 +31,13 @@ initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
by a string: the full Python path to the extension's class name. For example::
by a string: the full Python path to the extension's class name. For example:
.. code-block:: python
EXTENSIONS = {
'scrapy.extensions.corestats.CoreStats': 500,
'scrapy.extensions.telnet.TelnetConsole': 500,
"scrapy.extensions.corestats.CoreStats": 500,
"scrapy.extensions.telnet.TelnetConsole": 500,
}
@ -64,10 +66,12 @@ Disabling an extension
In order to disable an extension that comes enabled by default (i.e. those
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
``None``. For example::
``None``. For example:
.. code-block:: python
EXTENSIONS = {
'scrapy.extensions.corestats.CoreStats': None,
"scrapy.extensions.corestats.CoreStats": None,
}
Writing your own extension
@ -98,7 +102,9 @@ in the previous section. This extension will log a message every time:
The extension will be enabled through the ``MYEXT_ENABLED`` setting and the
number of items will be specified through the ``MYEXT_ITEMCOUNT`` setting.
Here is the code of such extension::
Here is the code of such extension:
.. code-block:: python
import logging
from scrapy import signals
@ -106,8 +112,8 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging:
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count
self.items_scraped = 0
@ -116,11 +122,11 @@ Here is the code of such extension::
def from_crawler(cls, crawler):
# first check if the extension should be enabled and raise
# NotConfigured otherwise
if not crawler.settings.getbool('MYEXT_ENABLED'):
if not crawler.settings.getbool("MYEXT_ENABLED"):
raise NotConfigured
# get the number of items from settings
item_count = crawler.settings.getint('MYEXT_ITEMCOUNT', 1000)
item_count = crawler.settings.getint("MYEXT_ITEMCOUNT", 1000)
# instantiate the extension object
ext = cls(item_count)

117
docs/topics/feed-exports.rst
View File

@ -58,7 +58,7 @@ CSV
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- To specify columns to export and their order use
- To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
@ -101,12 +101,12 @@ The storages backends supported out of the box are:
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-s3` (requires boto3_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
not available. For example, the S3 backend is only available if the boto3_
library is installed.
@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -193,7 +199,7 @@ The feeds are stored on `Amazon S3`_.
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
- Required external libraries: `boto3`_ >= 1.20.0
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@ -209,6 +215,12 @@ You can also define a custom ACL and custom endpoint for exported feeds using th
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -236,6 +248,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
@ -290,10 +308,11 @@ class, which is the default value of the ``item_filter`` :ref:`feed option <feed
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
method ``accepts`` and taking ``feed_options`` as an argument.
For instance::
For instance:
.. code-block:: python
class MyCustomFilter:
def __init__(self, feed_options):
self.feed_options = feed_options
@ -487,6 +506,8 @@ as a fallback value if that key is not provided for a specific feed definition:
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
- :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported)
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
.. versionadded:: 2.4.0
@ -515,6 +536,10 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
Use ``utf-8`` if you want UTF-8 for JSON too.
.. versionchanged:: 2.8
The :command:`startproject` command now sets this setting to
``utf-8`` in the generated ``settings.py`` file.
.. setting:: FEED_EXPORT_FIELDS
FEED_EXPORT_FIELDS
@ -522,18 +547,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
A list of fields to export, optional.
Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
defined in :ref:`item objects <topics-items>` yielded by your spider.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
order and their output names. See :attr:`BaseItemExporter.fields_to_export
<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
.. setting:: FEED_EXPORT_INDENT
@ -599,23 +615,27 @@ For a complete list of available values, access the `Canned ACL`_ section on Ama
FEED_STORAGES_BASE
------------------
Default::
Default:
.. code-block:: python
{
'': 'scrapy.extensions.feedexport.FileFeedStorage',
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
"": "scrapy.extensions.feedexport.FileFeedStorage",
"file": "scrapy.extensions.feedexport.FileFeedStorage",
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
}
A dict containing the built-in feed storage backends supported by Scrapy. You
can disable any of these backends by assigning ``None`` to their URI scheme in
:setting:`FEED_STORAGES`. E.g., to disable the built-in FTP storage backend
(without replacement), place this in your ``settings.py``::
(without replacement), place this in your ``settings.py``:
.. code-block:: python
FEED_STORAGES = {
'ftp': None,
"ftp": None,
}
.. setting:: FEED_EXPORTERS
@ -633,25 +653,30 @@ serialization formats and the values are paths to :ref:`Item exporter
FEED_EXPORTERS_BASE
-------------------
Default::
Default:
.. code-block:: python
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
'marshal': 'scrapy.exporters.MarshalItemExporter',
'pickle': 'scrapy.exporters.PickleItemExporter',
"json": "scrapy.exporters.JsonItemExporter",
"jsonlines": "scrapy.exporters.JsonLinesItemExporter",
"jsonl": "scrapy.exporters.JsonLinesItemExporter",
"jl": "scrapy.exporters.JsonLinesItemExporter",
"csv": "scrapy.exporters.CsvItemExporter",
"xml": "scrapy.exporters.XmlItemExporter",
"marshal": "scrapy.exporters.MarshalItemExporter",
"pickle": "scrapy.exporters.PickleItemExporter",
}
A dict containing the built-in feed exporters supported by Scrapy. You can
disable any of these exporters by assigning ``None`` to their serialization
format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
(without replacement), place this in your ``settings.py``::
(without replacement), place this in your ``settings.py``:
.. code-block:: python
FEED_EXPORTERS = {
'csv': None,
"csv": None,
}
@ -681,7 +706,9 @@ generated:
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
For instance, if your settings include:
.. code-block:: python
FEED_EXPORT_BATCH_ITEM_COUNT = 100
@ -750,24 +777,28 @@ The function signature should be as follows:
For example, to include the :attr:`name <scrapy.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
#. Define the following function somewhere in your project:
.. code-block:: python
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
return {**params, "spider_name": spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:
.. code-block:: python
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
FEED_URI_PARAMS = "myproject.utils.uri_params"
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _boto3: https://github.com/boto/boto3
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/

74
docs/topics/item-pipeline.rst
View File

@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
.. classmethod:: from_crawler(cls, crawler)
If present, this classmethod is called to create a pipeline instance
If present, this class method is called to create a pipeline instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the pipeline. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for pipeline to
@ -81,39 +81,44 @@ Price validation and dropping items with no prices
Let's take a look at the following hypothetical pipeline that adjusts the
``price`` attribute for those items that do not include VAT
(``price_excludes_vat`` attribute), and drops those items which don't
contain a price::
contain a price:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class PricePipeline:
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter.get('price'):
if adapter.get('price_excludes_vat'):
adapter['price'] = adapter['price'] * self.vat_factor
if adapter.get("price"):
if adapter.get("price_excludes_vat"):
adapter["price"] = adapter["price"] * self.vat_factor
return item
else:
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
--------------------------
Write items to a JSON lines file
--------------------------------
The following pipeline stores all scraped items (from all spiders) into a
single ``items.jl`` file, containing one item per line serialized in JSON
format::
single ``items.jsonl`` file, containing one item per line serialized in JSON
format:
.. code-block:: python
import json
from itemadapter import ItemAdapter
class JsonWriterPipeline:
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'w')
self.file = open("items.jsonl", "w")
def close_spider(self, spider):
self.file.close()
@ -135,14 +140,17 @@ MongoDB address and database name are specified in Scrapy settings;
MongoDB collection is named after item class.
The main point of this example is to show how to use :meth:`from_crawler`
method and how to clean up the resources properly.::
method and how to clean up the resources properly.
.. skip: next
.. code-block:: python
import pymongo
from itemadapter import ItemAdapter
class MongoPipeline:
collection_name = 'scrapy_items'
class MongoPipeline:
collection_name = "scrapy_items"
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
@ -151,8 +159,8 @@ method and how to clean up the resources properly.::
@classmethod
def from_crawler(cls, crawler):
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
mongo_uri=crawler.settings.get("MONGO_URI"),
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
)
def open_spider(self, spider):
@ -183,13 +191,15 @@ render a screenshot of the item URL. After the request response is downloaded,
the item pipeline saves the screenshot to a file and adds the filename to the
item.
::
.. code-block:: python
import hashlib
from pathlib import Path
from urllib.parse import quote
import scrapy
from itemadapter import ItemAdapter
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.defer import maybe_deferred_to_future
@ -203,8 +213,10 @@ item.
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await maybe_deferred_to_future(
spider.crawler.engine.download(request, spider)
)
if response.status != 200:
# Error happened, return item.
@ -214,8 +226,7 @@ item.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
# Store filename in item.
adapter["screenshot_filename"] = filename
@ -228,23 +239,24 @@ Duplicates filter
A filter that looks for duplicate items, and drops those items that were
already processed. Let's say that our items have a unique id, but our spider
returns multiples items with the same id::
returns multiples items with the same id:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class DuplicatesPipeline:
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
if adapter["id"] in self.ids_seen:
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(adapter['id'])
self.ids_seen.add(adapter["id"])
return item
@ -252,11 +264,13 @@ Activating an Item Pipeline component
=====================================
To activate an Item Pipeline component you must add its class to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
:setting:`ITEM_PIPELINES` setting, like in the following example:
.. code-block:: python
ITEM_PIPELINES = {
'myproject.pipelines.PricePipeline': 300,
'myproject.pipelines.JsonWriterPipeline': 800,
"myproject.pipelines.PricePipeline": 300,
"myproject.pipelines.JsonWriterPipeline": 800,
}
The integer values you assign to classes in this setting determine the

141
docs/topics/items.rst
View File

@ -76,10 +76,13 @@ make it the most feature-complete item type:
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Example::
Example:
.. code-block:: python
from scrapy.item import Item, Field
class CustomItem(Item):
one_field = Field()
another_field = Field()
@ -102,15 +105,13 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
They work natively in Python 3.7 or later, or using the `dataclasses
backport`_ in Python 3.6.
Example:
.. _dataclasses backport: https://pypi.org/project/dataclasses/
Example::
.. code-block:: python
from dataclasses import dataclass
@dataclass
class CustomItem:
one_field: str
@ -138,10 +139,13 @@ Additionally, ``attr.s`` items also allow to:
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
Example::
Example:
.. code-block:: python
import attr
@attr.s
class CustomItem:
one_field = attr.ib()
@ -157,10 +161,13 @@ Declaring Item subclasses
-------------------------
Item subclasses are declared using a simple class definition syntax and
:class:`Field` objects. Here is an example::
:class:`Field` objects. Here is an example:
.. code-block:: python
import scrapy
class Product(scrapy.Item):
name = scrapy.Field()
price = scrapy.Field()
@ -227,62 +234,68 @@ notice the API is very similar to the :class:`dict` API.
Creating items
''''''''''''''
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
.. code-block:: pycon
>>> product = Product(name="Desktop PC", price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
Getting field values
''''''''''''''''''''
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
.. code-block:: pycon
>>> product['price']
1000
>>> product["name"]
Desktop PC
>>> product.get("name")
Desktop PC
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product["price"]
1000
>>> product.get('last_updated', 'not set')
not set
>>> product["last_updated"]
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product.get("last_updated", "not set")
not set
>>> product.get('lala', 'unknown field')
'unknown field'
>>> product["lala"] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> 'name' in product # is name field populated?
True
>>> product.get("lala", "unknown field")
'unknown field'
>>> 'last_updated' in product # is last_updated populated?
False
>>> "name" in product # is name field populated?
True
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> "last_updated" in product # is last_updated populated?
False
>>> 'lala' in product.fields # is lala a declared field?
False
>>> "last_updated" in product.fields # is last_updated a declared field?
True
>>> "lala" in product.fields # is lala a declared field?
False
Setting field values
''''''''''''''''''''
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
.. code-block:: pycon
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> product["last_updated"] = "today"
>>> product["last_updated"]
today
>>> product["lala"] = "test" # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Accessing all populated values
@ -290,11 +303,13 @@ Accessing all populated values
To access all populated values, just use the typical :class:`dict` API:
>>> product.keys()
['price', 'name']
.. code-block:: pycon
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
>>> product.keys()
['price', 'name']
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
.. _copying-items:
@ -332,18 +347,20 @@ Other common tasks
Creating dicts from items:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
.. code-block:: pycon
Creating items from dicts:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
Creating items from dicts:
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> Product({"name": "Laptop PC", "price": 1500})
Product(price=1500, name='Laptop PC')
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Extending Item subclasses
@ -352,17 +369,21 @@ Extending Item subclasses
You can extend Items (to add more fields or to change some metadata for some
fields) by declaring a subclass of your original Item.
For example::
For example:
.. code-block:: python
class DiscountedProduct(Product):
discount_percent = scrapy.Field(serializer=str)
discount_expiration_date = scrapy.Field()
You can also extend field metadata by using the previous field metadata and
appending more values, or changing existing values, like this::
appending more values, or changing existing values, like this:
.. code-block:: python
class SpecificProduct(Product):
name = scrapy.Field(Product.fields['name'], serializer=my_serializer)
name = scrapy.Field(Product.fields["name"], serializer=my_serializer)
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
keeping all the previously existing metadata values.

6
docs/topics/jobs.rst
View File

@ -51,11 +51,13 @@ loading that attribute from the job directory, when the spider starts and
stops.
Here's an example of a callback that uses the spider state (other spider code
is omitted for brevity)::
is omitted for brevity):
.. code-block:: python
def parse_item(self, response):
# parse item here
self.state['items_count'] = self.state.get('items_count', 0) + 1
self.state["items_count"] = self.state.get("items_count", 0) + 1
Persistence gotchas
===================

98
docs/topics/leaks.rst
View File

@ -70,13 +70,15 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function::
telnet localhost 6023
>>> prefs()
Live References
.. code-block:: pycon
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
FormRequest 878 oldest: 7s ago
>>> prefs()
Live References
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
FormRequest 878 oldest: 7s ago
As you can see, that report also shows the "age" of the oldest object in each
class. If you're running multiple spiders per process chances are you can
@ -114,7 +116,9 @@ a priori, of course) by using the ``trackref`` tool.
After the crawler is running for a few minutes and we notice its memory usage
has grown a lot, we can enter its telnet console and check the live
references::
references:
.. code-block:: pycon
>>> prefs()
Live References
@ -134,19 +138,23 @@ generating the leaks (passing response references inside requests).
Sometimes extra information about live objects can be helpful.
Let's check the oldest response:
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
.. code-block:: pycon
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest("HtmlResponse")
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
If you want to iterate over all objects, instead of getting the oldest one, you
can use the :func:`scrapy.utils.trackref.iter_all` function:
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
.. code-block:: pycon
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all("HtmlResponse")]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
Too many spiders?
-----------------
@ -154,11 +162,13 @@ Too many spiders?
If your project has too many spiders executed in parallel,
the output of :func:`prefs()` can be difficult to read.
For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclases). For
ignore a particular class (and all its subclasses). For
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. code-block:: pycon
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. module:: scrapy.utils.trackref
:synopsis: Track references of live objects
@ -216,30 +226,32 @@ If you use ``pip``, you can install muppy with the following command::
Here's an example to view all Python objects available in
the heap using muppy:
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
.. code-block:: pycon
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
For more info about muppy, refer to the `muppy documentation`_.

10
docs/topics/link-extractors.rst
View File

@ -18,7 +18,9 @@ through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
variable in your spider, and use it from your spider callbacks:
.. code-block:: python
def parse(self, response):
for link in self.link_extractor.extract_links(response):
@ -132,10 +134,12 @@ LxmlLinkExtractor
.. highlight:: python
You can use the following function in ``process_value``::
You can use the following function in ``process_value``:
.. code-block:: python
def process_value(value):
m = re.search("javascript:goToPage\('(.*?)'", value)
m = re.search(r"javascript:goToPage\('(.*?)'", value)
if m:
return m.group(1)

116
docs/topics/loaders.rst
View File

@ -46,18 +46,21 @@ using a proper processing function.
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
chapter <topics-items>`::
chapter <topics-items>`:
.. code-block:: python
from scrapy.loader import ItemLoader
from myproject.items import Product
def parse(self, response):
l = ItemLoader(item=Product(), response=response)
l.add_xpath('name', '//div[@class="product_name"]')
l.add_xpath('name', '//div[@class="product_title"]')
l.add_xpath('price', '//p[@id="price"]')
l.add_css('stock', 'p#stock')
l.add_value('last_updated', 'today') # you can also use literal values
l.add_xpath("name", '//div[@class="product_name"]')
l.add_xpath("name", '//div[@class="product_title"]')
l.add_xpath("price", '//p[@id="price"]')
l.add_css("stock", "p#stock")
l.add_value("last_updated", "today") # you can also use literal values
return l.load_item()
By quickly looking at that code, we can see the ``name`` field is being
@ -93,11 +96,14 @@ will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
One approach to overcome this is to define items using the
:func:`~dataclasses.field` function, with a ``default`` argument::
:func:`~dataclasses.field` function, with a ``default`` argument:
.. code-block:: python
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: Optional[str] = field(default=None)
@ -122,14 +128,16 @@ processor). The result of the output processor is the final value that gets
assigned to the item.
Let's see an example to illustrate how the input and output processors are
called for a particular field (the same applies for any other field)::
called for a particular field (the same applies for any other field):
.. code-block:: python
l = ItemLoader(Product(), some_selector)
l.add_xpath('name', xpath1) # (1)
l.add_xpath('name', xpath2) # (2)
l.add_css('name', css) # (3)
l.add_value('name', 'test') # (4)
return l.load_item() # (5)
l.add_xpath("name", xpath1) # (1)
l.add_xpath("name", xpath2) # (2)
l.add_css("name", css) # (3)
l.add_value("name", "test") # (4)
return l.load_item() # (5)
So what happens is:
@ -184,13 +192,15 @@ processors <itemloaders:built-in-processors>` built-in for convenience.
Declaring Item Loaders
======================
Item Loaders are declared using a class definition syntax. Here is an example::
Item Loaders are declared using a class definition syntax. Here is an example:
.. code-block:: python
from itemloaders.processors import TakeFirst, MapCompose, Join
from scrapy.loader import ItemLoader
class ProductLoader(ItemLoader):
class ProductLoader(ItemLoader):
default_output_processor = TakeFirst()
name_in = MapCompose(str.title)
@ -215,16 +225,20 @@ As seen in the previous section, input and output processors can be declared in
the Item Loader definition, and it's very common to declare input processors
this way. However, there is one more place where you can specify the input and
output processors to use: in the :ref:`Item Field <topics-items-fields>`
metadata. Here is an example::
metadata. Here is an example:
.. code-block:: python
import scrapy
from itemloaders.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_price(value):
if value.isdigit():
return value
class Product(scrapy.Item):
name = scrapy.Field(
input_processor=MapCompose(remove_tags),
@ -235,12 +249,15 @@ metadata. Here is an example::
output_processor=TakeFirst(),
)
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': 'Welcome to my website', 'price': '1000'}
.. code-block:: pycon
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
>>> il.add_value("price", ["&euro;", "<span>1000</span>"])
>>> il.load_item()
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:
@ -263,10 +280,12 @@ declaring, instantiating or using Item Loader. They are used to modify the
behaviour of the input/output processors.
For example, suppose you have a function ``parse_length`` which receives a text
value and extracts a length from it::
value and extracts a length from it:
.. code-block:: python
def parse_length(text, loader_context):
unit = loader_context.get('unit', 'm')
unit = loader_context.get("unit", "m")
# ... length parsing code goes here ...
return parsed_length
@ -278,22 +297,28 @@ function (``parse_length`` in this case) can thus use them.
There are several ways to modify Item Loader context values:
1. By modifying the currently active Item Loader context
(:attr:`~ItemLoader.context` attribute)::
(:attr:`~ItemLoader.context` attribute):
.. code-block:: python
loader = ItemLoader(product)
loader.context['unit'] = 'cm'
loader.context["unit"] = "cm"
2. On Item Loader instantiation (the keyword arguments of Item Loader
``__init__`` method are stored in the Item Loader context)::
``__init__`` method are stored in the Item Loader context):
loader = ItemLoader(product, unit='cm')
.. code-block:: python
loader = ItemLoader(product, unit="cm")
3. On Item Loader declaration, for those input/output processors that support
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
them::
them:
.. code-block:: python
class ProductLoader(ItemLoader):
length_out = MapCompose(parse_length, unit='cm')
length_out = MapCompose(parse_length, unit="cm")
ItemLoader objects
@ -323,25 +348,29 @@ Example::
Without nested loaders, you need to specify the full xpath (or css) for each value
that you wish to extract.
Example::
Example:
.. code-block:: python
loader = ItemLoader(item=Item())
# load stuff not in the footer
loader.add_xpath('social', '//footer/a[@class = "social"]/@href')
loader.add_xpath('email', '//footer/a[@class = "email"]/@href')
loader.add_xpath("social", '//footer/a[@class = "social"]/@href')
loader.add_xpath("email", '//footer/a[@class = "email"]/@href')
loader.load_item()
Instead, you can create a nested loader with the footer selector and add values
relative to the footer. The functionality is the same but you avoid repeating
the footer selector.
Example::
Example:
.. code-block:: python
loader = ItemLoader(item=Item())
# load stuff not in the footer
footer_loader = loader.nested_xpath('//footer')
footer_loader.add_xpath('social', 'a[@class = "social"]/@href')
footer_loader.add_xpath('email', 'a[@class = "email"]/@href')
footer_loader = loader.nested_xpath("//footer")
footer_loader.add_xpath("social", 'a[@class = "social"]/@href')
footer_loader.add_xpath("email", 'a[@class = "email"]/@href')
# no need to call footer_loader.load_item()
loader.load_item()
@ -370,25 +399,32 @@ three dashes (e.g. ``---Plasma TV---``) and you don't want to end up scraping
those dashes in the final product names.
Here's how you can remove those dashes by reusing and extending the default
Product Item Loader (``ProductLoader``)::
Product Item Loader (``ProductLoader``):
.. code-block:: python
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
def strip_dashes(x):
return x.strip('-')
return x.strip("-")
class SiteSpecificLoader(ProductLoader):
name_in = MapCompose(strip_dashes, ProductLoader.name_in)
Another case where extending Item Loaders can be very helpful is when you have
multiple source formats, for example XML and HTML. In the XML version you may
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
want to remove ``CDATA`` occurrences. Here's an example of how to do it:
.. code-block:: python
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
from myproject.utils.xml import remove_cdata
class XmlProductLoader(ProductLoader):
name_in = MapCompose(remove_cdata, ProductLoader.name_in)

111
docs/topics/logging.rst
View File

@ -39,16 +39,22 @@ How to log messages
===================
Here's a quick example of how to log a message using the ``logging.WARNING``
level::
level:
.. code-block:: python
import logging
logging.warning("This is a warning")
There are shortcuts for issuing log messages on any of the standard 5 levels,
and there's also a general ``logging.log`` method which takes a given level as
argument. If needed, the last example could be rewritten as::
argument. If needed, the last example could be rewritten as:
.. code-block:: python
import logging
logging.log(logging.WARNING, "This is a warning")
On top of that, you can create different "loggers" to encapsulate messages. (For
@ -59,24 +65,33 @@ constructions.
The previous examples use the root logger behind the scenes, which is a top level
logger where all messages are propagated to (unless otherwise specified). Using
``logging`` helpers is merely a shortcut for getting the root logger
explicitly, so this is also an equivalent of the last snippets::
explicitly, so this is also an equivalent of the last snippets:
.. code-block:: python
import logging
logger = logging.getLogger()
logger.warning("This is a warning")
You can use a different logger just by getting its name with the
``logging.getLogger`` function::
``logging.getLogger`` function:
.. code-block:: python
import logging
logger = logging.getLogger('mycustomlogger')
logger = logging.getLogger("mycustomlogger")
logger.warning("This is a warning")
Finally, you can ensure having a custom logger for any module you're working on
by using the ``__name__`` variable, which is populated with current module's
path::
path:
.. code-block:: python
import logging
logger = logging.getLogger(__name__)
logger.warning("This is a warning")
@ -94,33 +109,37 @@ Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
instance, which can be accessed and used like this::
instance, which can be accessed and used like this:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapy.org']
class MySpider(scrapy.Spider):
name = "myspider"
start_urls = ["https://scrapy.org"]
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
self.logger.info("Parse function called on %s", response.url)
That logger is created using the Spider's name, but you can use any custom
Python logger you want. For example::
Python logger you want. For example:
.. code-block:: python
import logging
import scrapy
logger = logging.getLogger('mycustomlogger')
logger = logging.getLogger("mycustomlogger")
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapy.org']
name = "myspider"
start_urls = ["https://scrapy.org"]
def parse(self, response):
logger.info('Parse function called on %s', response.url)
logger.info("Parse function called on %s", response.url)
.. _topics-logging-configuration:
@ -229,7 +248,9 @@ the crawl.
Next, we can see that the message has INFO level. To hide it
we should set logging level for ``scrapy.spidermiddlewares.httperror``
higher than INFO; next level after INFO is WARNING. It could be done
e.g. in the spider's ``__init__`` method::
e.g. in the spider's ``__init__`` method:
.. code-block:: python
import logging
import scrapy
@ -238,7 +259,7 @@ e.g. in the spider's ``__init__`` method::
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('scrapy.spidermiddlewares.httperror')
logger = logging.getLogger("scrapy.spidermiddlewares.httperror")
logger.setLevel(logging.WARNING)
super().__init__(*args, **kwargs)
@ -249,43 +270,53 @@ You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
filter out unwanted messages:
.. code-block:: python
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
match = re.search(r"\d{3} [Ee]rror, retrying", record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
(middlewares, spider, etc.):
import logging
import scrapy
.. code-block:: python
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
and hide it without affecting other loggers:
.. code-block:: python
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger = logging.getLogger("my_logger")
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================
@ -306,14 +337,14 @@ scrapy.utils.log module
so it is recommended to only use :func:`logging.basicConfig` together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
This is an example on how to redirect ``INFO`` or higher messages to a file:
.. code-block:: python
import logging
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',
level=logging.INFO
filename="log.txt", format="%(levelname)s: %(message)s", level=logging.INFO
)
Refer to :ref:`run-from-script` for more details about using Scrapy this

208
docs/topics/media-pipeline.rst
View File

@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
@ -87,13 +87,17 @@ Enabling your Media Pipeline
To enable your media pipeline you must first add it to your project
:setting:`ITEM_PIPELINES` setting.
For Images Pipeline, use::
For Images Pipeline, use:
ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}
.. code-block:: python
For Files Pipeline, use::
ITEM_PIPELINES = {"scrapy.pipelines.images.ImagesPipeline": 1}
ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
For Files Pipeline, use:
.. code-block:: python
ITEM_PIPELINES = {"scrapy.pipelines.files.FilesPipeline": 1}
.. note::
You can also use both the Files and Images Pipeline at the same time.
@ -103,13 +107,17 @@ Then, configure the target storage setting to a valid value that will be used
for storing the downloaded images. Otherwise the pipeline will remain disabled,
even if you include it in the :setting:`ITEM_PIPELINES` setting.
For the Files Pipeline, set the :setting:`FILES_STORE` setting::
For the Files Pipeline, set the :setting:`FILES_STORE` setting:
FILES_STORE = '/path/to/valid/dir'
.. code-block:: python
For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
FILES_STORE = "/path/to/valid/dir"
IMAGES_STORE = '/path/to/valid/dir'
For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:
.. code-block:: python
IMAGES_STORE = "/path/to/valid/dir"
.. _topics-file-naming:
@ -156,12 +164,12 @@ By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
image_perspective = request.url.split('/')[-2]
image_filename = f'{image_url_hash}_{image_perspective}.jpg'
image_perspective = request.url.split("/")[-2]
image_filename = f"{image_url_hash}_{image_perspective}.jpg"
return image_filename
@ -234,30 +242,38 @@ If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
For example, this is a valid :setting:`IMAGES_STORE` value:
IMAGES_STORE = 's3://bucket/images'
.. code-block:: python
IMAGES_STORE = "s3://bucket/images"
You can modify the Access Control List (ACL) policy used for the stored files,
which is defined by the :setting:`FILES_STORE_S3_ACL` and
:setting:`IMAGES_STORE_S3_ACL` settings. By default, the ACL is set to
``private``. To make the files publicly available use the ``public-read``
policy::
policy:
IMAGES_STORE_S3_ACL = 'public-read'
.. code-block:: python
IMAGES_STORE_S3_ACL = "public-read"
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
settings:
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
.. code-block:: python
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
AWS_ENDPOINT_URL = "http://minio.example.com:9000"
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:
.. code-block:: python
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
@ -278,10 +294,12 @@ bucket. Scrapy will automatically upload the files to the bucket. (requires `goo
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:
IMAGES_STORE = 'gs://bucket/images/'
GCS_PROJECT_ID = 'project_id'
.. code-block:: python
IMAGES_STORE = "gs://bucket/images/"
GCS_PROJECT_ID = "project_id"
For information about authentication, see this `documentation`_.
@ -292,9 +310,11 @@ which is defined by the :setting:`FILES_STORE_GCS_ACL` and
:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
To make the files publicly available use the ``publicRead``
policy::
policy:
IMAGES_STORE_GCS_ACL = 'publicRead'
.. code-block:: python
IMAGES_STORE_GCS_ACL = "publicRead"
For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
@ -319,10 +339,13 @@ respectively), the pipeline will put the results under the respective field
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.Item` class::
``images`` field. For instance, using the :class:`~scrapy.Item` class:
.. code-block:: python
import scrapy
class MyItem(scrapy.Item):
# ... other item fields ...
image_urls = scrapy.Field()
@ -332,16 +355,20 @@ If you want to use another field name for the URLs key or for the results key,
it is also possible to override it.
For the Files Pipeline, set :setting:`FILES_URLS_FIELD` and/or
:setting:`FILES_RESULT_FIELD` settings::
:setting:`FILES_RESULT_FIELD` settings:
FILES_URLS_FIELD = 'field_name_for_your_files_urls'
FILES_RESULT_FIELD = 'field_name_for_your_processed_files'
.. code-block:: python
FILES_URLS_FIELD = "field_name_for_your_files_urls"
FILES_RESULT_FIELD = "field_name_for_your_processed_files"
For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
:setting:`IMAGES_RESULT_FIELD` settings::
:setting:`IMAGES_RESULT_FIELD` settings:
IMAGES_URLS_FIELD = 'field_name_for_your_images_urls'
IMAGES_RESULT_FIELD = 'field_name_for_your_processed_images'
.. code-block:: python
IMAGES_URLS_FIELD = "field_name_for_your_images_urls"
IMAGES_RESULT_FIELD = "field_name_for_your_processed_images"
If you need something more complex and want to override the custom pipeline
behaviour, see :ref:`topics-media-pipeline-override`.
@ -367,7 +394,9 @@ File expiration
The Image Pipeline avoids downloading files that were downloaded recently. To
adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or
:setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which
specifies the delay in number of days::
specifies the delay in number of days:
.. code-block:: python
# 120 days of delay for files expiration
FILES_EXPIRES = 120
@ -401,11 +430,13 @@ images.
In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
where the keys are the thumbnail names and the values are their dimensions.
For example::
For example:
.. code-block:: python
IMAGES_THUMBS = {
'small': (50, 50),
'big': (270, 270),
"small": (50, 50),
"big": (270, 270),
}
When you use this feature, the Images Pipeline will create thumbnails of the
@ -496,17 +527,19 @@ See here the methods that you can override in your custom Files Pipeline:
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
original filenames (e.g. ``files/foo.png``):
import os
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
class MyFilesPipeline(FilesPipeline):
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return "files/" + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -522,13 +555,16 @@ See here the methods that you can override in your custom Files Pipeline:
As seen on the workflow, the pipeline will get the URLs of the images to
download from the item. In order to do this, you can override the
:meth:`~get_media_requests` method and return a Request for each
file URL::
file URL:
.. code-block:: python
from itemadapter import ItemAdapter
def get_media_requests(self, item, info):
adapter = ItemAdapter(item)
for file_url in adapter['file_urls']:
for file_url in adapter["file_urls"]:
yield scrapy.Request(file_url)
Those requests will be processed by the pipeline and, when they have finished
@ -568,15 +604,26 @@ See here the methods that you can override in your custom Files Pipeline:
guaranteed to retain the same order of the requests returned from the
:meth:`~get_media_requests` method.
Here's a typical value of the ``results`` argument::
Here's a typical value of the ``results`` argument:
[(True,
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
'url': 'http://www.example.com/files/product1.pdf',
'status': 'downloaded'}),
(False,
Failure(...))]
.. invisible-code-block: python
from twisted.python.failure import Failure
.. code-block:: python
[
(
True,
{
"checksum": "2b00042f7481c7b056c4b410d28f33cf",
"path": "full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg",
"url": "http://www.example.com/files/product1.pdf",
"status": "downloaded",
},
),
(False, Failure(...)),
]
By default the :meth:`get_media_requests` method returns ``None`` which
means there are no files to download for the item.
@ -593,17 +640,20 @@ See here the methods that you can override in your custom Files Pipeline:
Here is an example of the :meth:`~item_completed` method where we
store the downloaded file paths (passed in results) in the ``file_paths``
item field, and we drop the item if it doesn't contain any files::
item field, and we drop the item if it doesn't contain any files:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
def item_completed(self, results, item, info):
file_paths = [x['path'] for ok, x in results if ok]
file_paths = [x["path"] for ok, x in results if ok]
if not file_paths:
raise DropItem("Item contains no files")
adapter = ItemAdapter(item)
adapter['file_paths'] = file_paths
adapter["file_paths"] = file_paths
return item
By default, the :meth:`item_completed` method returns the item.
@ -635,17 +685,19 @@ See here the methods that you can override in your custom Images Pipeline:
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
original filenames (e.g. ``files/foo.png``):
import os
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return "files/" + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -656,6 +708,26 @@ See here the methods that you can override in your custom Images Pipeline:
.. versionadded:: 2.4
The *item* parameter.
.. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
thumbnail download path of the image originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>`,
``thumb_id``,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.Item>`.
You can override this method to customize the thumbnail download path of each image.
You can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`thumb_path` method returns
``thumbs/<size name>/<request URL hash>.<extension>``.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
@ -681,33 +753,35 @@ Custom Images pipeline example
==============================
Here is a full example of the Images Pipeline whose methods are exemplified
above::
above:
.. code-block:: python
import scrapy
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
class MyImagesPipeline(ImagesPipeline):
def get_media_requests(self, item, info):
for image_url in item['image_urls']:
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def item_completed(self, results, item, info):
image_paths = [x['path'] for ok, x in results if ok]
image_paths = [x["path"] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
adapter = ItemAdapter(item)
adapter['image_paths'] = image_paths
adapter["image_paths"] = image_paths
return item
To enable your custom media pipeline component you must add its class import path to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
:setting:`ITEM_PIPELINES` setting, like in the following example:
ITEM_PIPELINES = {
'myproject.pipelines.MyImagesPipeline': 300
}
.. code-block:: python
ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5

59
docs/topics/practices.rst
View File

@ -7,6 +7,8 @@ Common Practices
This section documents common practices when using Scrapy. These are things
that cover many topics and don't often fall into any other specific section.
.. skip: start
.. _run-from-script:
Run Scrapy from a script
@ -25,23 +27,27 @@ the one used by all Scrapy commands.
Here's an example showing how to run a single spider with it.
::
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerProcess
class MySpider(scrapy.Spider):
# Your spider definition
...
process = CrawlerProcess(settings={
"FEEDS": {
"items.json": {"format": "json"},
},
})
process = CrawlerProcess(
settings={
"FEEDS": {
"items.json": {"format": "json"},
},
}
)
process.crawl(MySpider)
process.start() # the script will block here until the crawling is finished
process.start() # the script will block here until the crawling is finished
Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
documentation to get acquainted with its usage details.
@ -55,7 +61,7 @@ instance with your project settings.
What follows is a working example of how to do that, using the `testspiders`_
project as example.
::
.. code-block:: python
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
@ -63,8 +69,8 @@ project as example.
process = CrawlerProcess(get_project_settings())
# 'followall' is the name of one of the spiders of the project.
process.crawl('followall', domain='scrapy.org')
process.start() # the script will block here until the crawling is finished
process.crawl("followall", domain="scrapy.org")
process.start() # the script will block here until the crawling is finished
There's another Scrapy utility that provides more control over the crawling
process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper
@ -84,23 +90,25 @@ returned by the :meth:`CrawlerRunner.crawl
Here's an example of its usage, along with a callback to manually stop the
reactor after ``MySpider`` has finished running.
::
.. code-block:: python
from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class MySpider(scrapy.Spider):
# Your spider definition
...
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
runner = CrawlerRunner()
d = runner.crawl(MySpider)
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
reactor.run() # the script will block here until the crawling is finished
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
@ -115,29 +123,32 @@ the :ref:`internal API <topics-api>`.
Here is an example that runs multiple spiders simultaneously:
::
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
settings = get_project_settings()
process = CrawlerProcess(settings)
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start() # the script will block here until all crawling jobs are finished
process.start() # the script will block here until all crawling jobs are finished
Same example using :class:`~scrapy.crawler.CrawlerRunner`:
::
.. code-block:: python
import scrapy
from twisted.internet import reactor
@ -145,14 +156,17 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
configure_logging()
settings = get_project_settings()
runner = CrawlerRunner(settings)
@ -161,37 +175,42 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
d = runner.join()
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until all crawling jobs are finished
reactor.run() # the script will block here until all crawling jobs are finished
Same example but running the spiders sequentially by chaining the deferreds:
::
.. code-block:: python
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
settings = get_project_settings()
configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
def crawl():
yield runner.crawl(MySpider1)
yield runner.crawl(MySpider2)
reactor.stop()
crawl()
reactor.run() # the script will block here until the last crawl call is finished
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some
@ -214,6 +233,8 @@ different for different settings:
.. seealso:: :ref:`run-from-script`.
.. skip: end
.. _distributed-crawls:
Distributed crawls

476
docs/topics/request-response.rst
View File

@ -32,11 +32,20 @@ Request objects
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
request (once it's downloaded) as its first parameter.
In addition to a function, the following values are supported:
- ``None`` (default), which indicates that the spider's
:meth:`~scrapy.Spider.parse` method must be used.
- :func:`~scrapy.http.request.NO_CALLBACK`
For more information, see
:ref:`topics-request-response-ref-request-callback-arguments`.
.. note:: If exceptions are raised during processing, ``errback`` is
called instead.
:type callback: collections.abc.Callable
@ -67,18 +76,34 @@ Request objects
:param cookies: the request cookies. These can be sent in two forms.
1. Using a dict::
.. invisible-code-block: python
request_with_cookies = Request(url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'})
from scrapy.http import Request
2. Using a list of dicts::
1. Using a dict:
request_with_cookies = Request(url="http://www.example.com",
cookies=[{'name': 'currency',
'value': 'USD',
'domain': 'example.com',
'path': '/currency'}])
.. code-block:: python
request_with_cookies = Request(
url="http://www.example.com",
cookies={"currency": "USD", "country": "UY"},
)
2. Using a list of dicts:
.. code-block:: python
request_with_cookies = Request(
url="http://www.example.com",
cookies=[
{
"name": "currency",
"value": "USD",
"domain": "example.com",
"path": "/currency",
},
],
)
The latter form allows for customizing the ``domain`` and ``path``
attributes of the cookie. This is only useful if the cookies are saved
@ -95,12 +120,14 @@ Request objects
in :attr:`request.meta <scrapy.Request.meta>`.
Example of a request that sends manually-defined cookies and ignores
cookie storage::
cookie storage:
.. code-block:: python
Request(
url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'},
meta={'dont_merge_cookies': True},
cookies={"currency": "USD", "country": "UY"},
meta={"dont_merge_cookies": True},
)
For more info see :ref:`cookies-mw`.
@ -228,6 +255,8 @@ Request objects
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.http.request.NO_CALLBACK
.. autofunction:: scrapy.utils.request.request_from_dict
@ -240,11 +269,15 @@ The callback of a request is a function that will be called when the response
of that request is downloaded. The callback function will be called with the
downloaded :class:`Response` object as its first argument.
Example::
Example:
.. code-block:: python
def parse_page1(self, response):
return scrapy.Request("http://www.example.com/some_page.html",
callback=self.parse_page2)
return scrapy.Request(
"http://www.example.com/some_page.html", callback=self.parse_page2
)
def parse_page2(self, response):
# this would log http://www.example.com/some_page.html
@ -255,15 +288,18 @@ functions so you can receive the arguments later, in the second callback.
The following example shows how to achieve this by using the
:attr:`Request.cb_kwargs` attribute:
::
.. code-block:: python
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url))
request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url),
)
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
yield request
def parse_page2(self, response, main_url, foo):
yield dict(
main_url=main_url,
@ -289,7 +325,9 @@ It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
errors if needed:
.. code-block:: python
import scrapy
@ -297,24 +335,28 @@ errors if needed::
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
class ErrbackSpider(scrapy.Spider):
name = "errback_example"
start_urls = [
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
]
def start_requests(self):
for u in self.start_urls:
yield scrapy.Request(u, callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True)
yield scrapy.Request(
u,
callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True,
)
def parse_httpbin(self, response):
self.logger.info('Got successful response from {}'.format(response.url))
self.logger.info("Got successful response from {}".format(response.url))
# do something useful here...
def errback_httpbin(self, failure):
@ -328,16 +370,17 @@ errors if needed::
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
self.logger.error('HttpError on %s', response.url)
self.logger.error("HttpError on %s", response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error('DNSLookupError on %s', request.url)
self.logger.error("DNSLookupError on %s", request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
self.logger.error("TimeoutError on %s", request.url)
.. _errback-cb_kwargs:
@ -347,23 +390,309 @@ Accessing additional data in errback functions
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``::
achieve this by using ``Failure.request.cb_kwargs``:
.. code-block:: python
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url))
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url),
)
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs['main_url'],
main_url=failure.request.cb_kwargs["main_url"],
)
.. _request-fingerprints:
Request fingerprints
--------------------
There are some aspects of scraping, such as filtering out duplicate requests
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
unique identifier from a :class:`~scrapy.http.Request` object: a request
fingerprint.
You often do not need to worry about request fingerprints, the default request
fingerprinter works for most projects.
However, there is no universal way to generate a unique identifier from a
request, because different situations require comparing requests differently.
For example, sometimes you may need to compare URLs case-insensitively, include
URL fragments, exclude certain URL query parameters, include some or all
headers, etc.
To change how request fingerprints are built for your requests, use the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
.. setting:: REQUEST_FINGERPRINTER_CLASS
REQUEST_FINGERPRINTER_CLASS
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 2.7
Default: :class:`scrapy.utils.request.RequestFingerprinter`
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
import path.
.. autoclass:: scrapy.utils.request.RequestFingerprinter
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
REQUEST_FINGERPRINTER_IMPLEMENTATION
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 2.7
Default: ``'2.6'``
Determines which request fingerprinting algorithm is used by the default
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
Possible values are:
- ``'2.6'`` (default)
This implementation uses the same request fingerprinting algorithm as
Scrapy 2.6 and earlier versions.
Even though this is the default value for backward compatibility reasons,
it is a deprecated value.
- ``'2.7'``
This implementation was introduced in Scrapy 2.7 to fix an issue of the
previous implementation.
New projects should use this value. The :command:`startproject` command
sets this value in the generated ``settings.py`` file.
If you are using the default value (``'2.6'``) for this setting, and you are
using Scrapy components where changing the request fingerprinting algorithm
would cause undesired results, you need to carefully decide when to change the
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
setting to a custom request fingerprinter class that implements the 2.6 request
fingerprinting algorithm and does not log this warning (
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
class).
Scenarios where changing the request fingerprinting algorithm may cause
undesired results include, for example, using the HTTP cache middleware (see
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
Changing the request fingerprinting algorithm would invalidate the current
cache, requiring you to redownload all requests again.
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
your settings to switch already to the request fingerprinting implementation
that will be the only request fingerprinting implementation available in a
future version of Scrapy, and remove the deprecation warning triggered by using
the default value (``'2.6'``).
.. _2.6-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A request fingerprinter is a class that must implement the following method:
.. currentmodule:: None
.. method:: fingerprint(self, request)
Return a :class:`bytes` object that uniquely identifies *request*.
See also :ref:`request-fingerprint-restrictions`.
:param request: request to fingerprint
:type request: scrapy.http.Request
Additionally, it may also implement the following methods:
.. classmethod:: from_crawler(cls, crawler)
:noindex:
If present, this class method is called to create a request fingerprinter
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
new instance of the request fingerprinter.
*crawler* provides access to all Scrapy core components like settings and
signals; it is a way for the request fingerprinter to access them and hook
its functionality into Scrapy.
:param crawler: crawler that uses this request fingerprinter
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. classmethod:: from_settings(cls, settings)
If present, and ``from_crawler`` is not defined, this class method is called
to create a request fingerprinter instance from a
:class:`~scrapy.settings.Settings` object. It must return a new instance of
the request fingerprinter.
.. currentmodule:: scrapy.http
The :meth:`fingerprint` method of the default request fingerprinter,
:class:`scrapy.utils.request.RequestFingerprinter`, uses
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
For example, to take the value of a request header named ``X-ID`` into
account:
.. code-block:: python
# my_project/settings.py
REQUEST_FINGERPRINTER_CLASS = "my_project.utils.RequestFingerprinter"
# my_project/utils.py
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
def fingerprint(self, request):
return fingerprint(request, include_headers=["X-ID"])
You can also write your own fingerprinting logic from scratch.
However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
- Caching saves CPU by ensuring that fingerprints are calculated only once
per request, and not once per Scrapy component that needs the fingerprint
of a request.
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
request objects do not stay in memory forever just because you have
references to them in your cache dictionary.
For example, to take into account only the URL of a request, without any prior
URL canonicalization or taking the request method or body into account:
.. code-block:: python
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.url))
self.cache[request] = fp.digest()
return self.cache[request]
If you need to be able to override the request fingerprinting for arbitrary
requests from your spider callbacks, you may implement a request fingerprinter
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
when available, and then falls back to
:func:`scrapy.utils.request.fingerprint`. For example:
.. code-block:: python
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
def fingerprint(self, request):
if "fingerprint" in request.meta:
return request.meta["fingerprint"]
return fingerprint(request)
If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
without using the deprecated ``'2.6'`` value of the
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
request fingerprinter:
.. code-block:: python
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
from w3lib.url import canonicalize_url
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b"")
self.cache[request] = fp.digest()
return self.cache[request]
.. _request-fingerprint-restrictions:
Request fingerprint restrictions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Scrapy components that use request fingerprints may impose additional
restrictions on the format of the fingerprints that your :ref:`request
fingerprinter <custom-request-fingerprinter>` generates.
The following built-in Scrapy components have such restrictions:
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
value of :setting:`HTTPCACHE_STORAGE`)
Request fingerprints must be at least 1 byte long.
Path and filename length limits of the file system of
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
the following directory structure is created:
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
- first byte of a request fingerprint as hexadecimal
- fingerprint as hexadecimal
- filenames up to 16 characters long
For example, if a request fingerprint is made of 20 bytes (default),
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
and the name of your spider is ``'my_spider'`` your file system must
support a file path like::
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
The underlying DBM implementation must support keys as long as twice
the number of bytes of a request fingerprint, plus 5. For example,
if a request fingerprint is made of 20 bytes (default),
45-character-long keys must be supported.
.. _topics-request-meta:
Request.meta special keys
@ -445,7 +774,9 @@ Stopping the download of a Response
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signals will stop the download of a given response. See the following example::
signals will stop the download of a given response. See the following example:
.. code-block:: python
import scrapy
@ -457,7 +788,9 @@ signals will stop the download of a given response. See the following example::
@classmethod
def from_crawler(cls, crawler):
spider = super().from_crawler(crawler)
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
crawler.signals.connect(
spider.on_bytes_received, signal=scrapy.signals.bytes_received
)
return spider
def parse(self, response):
@ -586,11 +919,18 @@ Using FormRequest to send data via HTTP POST
If you want to simulate a HTML Form POST in your spider and send a couple of
key-value fields, you can return a :class:`FormRequest` object (from your
spider) like this::
spider) like this:
return [FormRequest(url="http://www.example.com/post/action",
formdata={'name': 'John Doe', 'age': '27'},
callback=self.after_post)]
.. skip: next
.. code-block:: python
return [
FormRequest(
url="http://www.example.com/post/action",
formdata={"name": "John Doe", "age": "27"},
callback=self.after_post,
)
]
.. _topics-request-response-ref-request-userlogin:
@ -602,25 +942,28 @@ type="hidden">`` elements, such as session related data or authentication
tokens (for login pages). When scraping, you'll want these fields to be
automatically pre-populated and only override a couple of them, such as the
user name and password. You can use the :meth:`FormRequest.from_response`
method for this job. Here's an example spider which uses it::
method for this job. Here's an example spider which uses it:
.. code-block:: python
import scrapy
def authentication_failed(response):
# TODO: Check the contents of the response and return True if it failed
# or False if it succeeded.
pass
class LoginSpider(scrapy.Spider):
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
name = "example.com"
start_urls = ["http://www.example.com/users/login.php"]
def parse(self, response):
return scrapy.FormRequest.from_response(
response,
formdata={'username': 'john', 'password': 'secret'},
callback=self.after_login
formdata={"username": "john", "password": "secret"},
callback=self.after_login,
)
def after_login(self, response):
@ -660,13 +1003,16 @@ dealing with JSON requests.
JsonRequest usage example
-------------------------
Sending a JSON POST request with a JSON payload::
Sending a JSON POST request with a JSON payload:
.. skip: next
.. code-block:: python
data = {
'name1': 'value1',
'name2': 'value2',
"name1": "value1",
"name2": "value2",
}
yield JsonRequest(url='http://www.example.com/post/action', data=data)
yield JsonRequest(url="http://www.example.com/post/action", data=data)
Response objects
@ -901,8 +1247,10 @@ TextResponse objects
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. code-block:: pycon
>>> str(b"body")
"b'body'"
.. attribute:: TextResponse.encoding
@ -933,6 +1281,12 @@ TextResponse objects
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
.. method:: TextResponse.jmespath(query)
A shortcut to ``TextResponse.selector.jmespath(query)``::
response.jmespath('object.[*]')
.. method:: TextResponse.xpath(query)
A shortcut to ``TextResponse.selector.xpath(query)``::

664
docs/topics/selectors.rst
View File

File diff suppressed because it is too large Load Diff

304
docs/topics/settings.rst
View File

@ -67,13 +67,18 @@ Example::
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
own settings that will take precedence and override the project ones. They can
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
custom_settings = {
'SOME_SETTING': 'some value',
"SOME_SETTING": "some value",
}
3. Project settings module
@ -98,6 +103,10 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Compatibility with pickle
=========================
Setting values must be :ref:`picklable <pickle-picklable>`.
Import paths and classes
========================
@ -111,14 +120,18 @@ class or a function, there are two different ways you can specify that object:
- As the object itself
For example::
For example:
.. skip: next
.. code-block:: python
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
"mybot.pipelines.validate.ValidateMyItem": 300,
}
.. note:: Passing non-callable objects is not supported.
@ -129,11 +142,13 @@ How to access settings
.. highlight:: python
In a spider, the settings are available through ``self.settings``::
In a spider, the settings are available through ``self.settings``:
.. code-block:: python
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://example.com']
name = "myspider"
start_urls = ["http://example.com"]
def parse(self, response):
print(f"Existing settings: {self.settings.attributes.keys()}")
@ -146,7 +161,9 @@ In a spider, the settings are available through ``self.settings``::
Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
extensions, middlewares and item pipelines::
extensions, middlewares and item pipelines:
.. code-block:: python
class MyExtension:
def __init__(self, log_is_enabled=False):
@ -156,7 +173,7 @@ extensions, middlewares and item pipelines::
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
return cls(settings.getbool('LOG_ENABLED'))
return cls(settings.getbool("LOG_ENABLED"))
The settings object can be used like a dict (e.g.,
``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting
@ -361,11 +378,13 @@ Scrapy shell <topics-shell>`.
DEFAULT_REQUEST_HEADERS
-----------------------
Default::
Default:
.. code-block:: python
{
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en",
}
The default headers used for Scrapy HTTP Requests. They're populated in the
@ -400,9 +419,12 @@ Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
a :class:`~scrapy.Request` based on its depth.
The priority of a request is adjusted as follows::
The priority of a request is adjusted as follows:
request.priority = request.priority - ( depth * DEPTH_PRIORITY )
.. skip: next
.. code-block:: python
request.priority = request.priority - (depth * DEPTH_PRIORITY)
As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
priority (BFO), while negative values increase request priority (DFO). See
@ -560,7 +582,6 @@ This setting must be one of these string values:
set this if you want the behavior of Scrapy<1.1
- ``'TLSv1.1'``: forces TLS version 1.1
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -592,23 +613,25 @@ orders. For more info see :ref:`topics-downloader-middleware-setting`.
DOWNLOADER_MIDDLEWARES_BASE
---------------------------
Default::
Default:
.. code-block:: python
{
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
"scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
"scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
"scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
"scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560,
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
"scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700,
"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
"scrapy.downloadermiddlewares.stats.DownloaderStats": 850,
"scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900,
}
A dict containing the downloader middlewares enabled by default in Scrapy. Low
@ -633,24 +656,41 @@ DOWNLOAD_DELAY
Default: ``0``
The amount of time (in secs) that the downloader should wait before downloading
consecutive pages from the same website. This can be used to throttle the
crawling speed to avoid hitting servers too hard. Decimal numbers are
supported. Example::
Minimum seconds to wait between 2 consecutive requests to the same domain.
DOWNLOAD_DELAY = 0.25 # 250 ms of delay
Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
servers too hard.
Decimal numbers are supported. For example, to send a maximum of 4 requests
every 10 seconds::
DOWNLOAD_DELAY = 2.5
This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
setting (which is enabled by default). By default, Scrapy doesn't wait a fixed
amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`.
setting, which is enabled by default.
When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
per ip address instead of per domain.
per IP address instead of per domain.
Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
concurrency for that domain is 1. When testing throttling configurations, it
usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first,
and only increase :setting:`DOWNLOAD_DELAY` once
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is
desired.
.. _spider-download_delay-attribute:
You can also change this setting per spider by setting ``download_delay``
spider attribute.
.. note::
This delay can be set per spider using :attr:`download_delay` spider attribute.
It is also possible to change this setting per domain, although it requires
non-trivial code. See the implementation of the :ref:`AutoThrottle
<topics-autothrottle>` extension for an example.
.. setting:: DOWNLOAD_HANDLERS
@ -667,15 +707,17 @@ See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
DOWNLOAD_HANDLERS_BASE
----------------------
Default::
Default:
.. code-block:: python
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
"file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",
"http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
"https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
"s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
}
@ -685,10 +727,12 @@ You should never modify this setting in your project, modify
You can disable any of these download handlers by assigning ``None`` to their
URI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP
handler (without replacement), place this in your ``settings.py``::
handler (without replacement), place this in your ``settings.py``:
.. code-block:: python
DOWNLOAD_HANDLERS = {
'ftp': None,
"ftp": None,
}
.. _http2:
@ -698,10 +742,12 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
#. Update :setting:`DOWNLOAD_HANDLERS` as follows:
.. code-block:: python
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
}
.. warning::
@ -730,6 +776,31 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
.. setting:: DOWNLOAD_SLOTS
DOWNLOAD_SLOTS
----------------
Default: ``{}``
Allows to define concurrency/delay parameters on per slot (domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
"quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
.. note::
For other downloader slots default settings values will be used:
- :setting:`DOWNLOAD_DELAY`: ``delay``
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@ -825,12 +896,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
The class used to detect and filter duplicate requests.
The default (``RFPDupeFilter``) filters based on request fingerprint using
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
override its ``request_fingerprint`` method. This method should accept
scrapy :class:`~scrapy.Request` object and return its fingerprint
(a string).
The default (``RFPDupeFilter``) filters based on the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
@ -874,18 +941,20 @@ A dict containing the extensions enabled in your project, and their orders.
EXTENSIONS_BASE
---------------
Default::
Default:
.. code-block:: python
{
'scrapy.extensions.corestats.CoreStats': 0,
'scrapy.extensions.telnet.TelnetConsole': 0,
'scrapy.extensions.memusage.MemoryUsage': 0,
'scrapy.extensions.memdebug.MemoryDebugger': 0,
'scrapy.extensions.closespider.CloseSpider': 0,
'scrapy.extensions.feedexport.FeedExporter': 0,
'scrapy.extensions.logstats.LogStats': 0,
'scrapy.extensions.spiderstate.SpiderState': 0,
'scrapy.extensions.throttle.AutoThrottle': 0,
"scrapy.extensions.corestats.CoreStats": 0,
"scrapy.extensions.telnet.TelnetConsole": 0,
"scrapy.extensions.memusage.MemoryUsage": 0,
"scrapy.extensions.memdebug.MemoryDebugger": 0,
"scrapy.extensions.closespider.CloseSpider": 0,
"scrapy.extensions.feedexport.FeedExporter": 0,
"scrapy.extensions.logstats.LogStats": 0,
"scrapy.extensions.spiderstate.SpiderState": 0,
"scrapy.extensions.throttle.AutoThrottle": 0,
}
A dict containing the extensions available by default in Scrapy, and their
@ -972,11 +1041,13 @@ A dict containing the item pipelines to use, and their orders. Order values are
arbitrary, but it is customary to define them in the 0-1000 range. Lower orders
process before higher orders.
Example::
Example:
.. code-block:: python
ITEM_PIPELINES = {
'mybot.pipelines.validate.ValidateMyItem': 300,
'mybot.pipelines.validate.StoreMyItem': 800,
"mybot.pipelines.validate.ValidateMyItem": 300,
"mybot.pipelines.validate.StoreMyItem": 800,
}
.. setting:: ITEM_PIPELINES_BASE
@ -1044,7 +1115,7 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
:ref:`Python logging documentation <logrecord-attributes>` for the whole
list of available placeholders.
.. setting:: LOG_DATEFORMAT
@ -1401,12 +1472,14 @@ testing spiders. For more info see :ref:`topics-contracts`.
SPIDER_CONTRACTS_BASE
---------------------
Default::
Default:
.. code-block:: python
{
'scrapy.contracts.default.UrlContract' : 1,
'scrapy.contracts.default.ReturnsContract': 2,
'scrapy.contracts.default.ScrapesContract': 3,
"scrapy.contracts.default.UrlContract": 1,
"scrapy.contracts.default.ReturnsContract": 2,
"scrapy.contracts.default.ScrapesContract": 3,
}
A dict containing the Scrapy contracts enabled by default in Scrapy. You should
@ -1415,10 +1488,12 @@ instead. For more info see :ref:`topics-contracts`.
You can disable any of these contracts by assigning ``None`` to their class
path in :setting:`SPIDER_CONTRACTS`. E.g., to disable the built-in
``ScrapesContract``, place this in your ``settings.py``::
``ScrapesContract``, place this in your ``settings.py``:
.. code-block:: python
SPIDER_CONTRACTS = {
'scrapy.contracts.default.ScrapesContract': None,
"scrapy.contracts.default.ScrapesContract": None,
}
.. setting:: SPIDER_LOADER_CLASS
@ -1467,14 +1542,16 @@ orders. For more info see :ref:`topics-spider-middleware-setting`.
SPIDER_MIDDLEWARES_BASE
-----------------------
Default::
Default:
.. code-block:: python
{
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
}
A dict containing the spider middlewares enabled by default in Scrapy, and
@ -1490,9 +1567,11 @@ Default: ``[]``
A list of modules where Scrapy will look for spiders.
Example::
Example:
SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev']
.. code-block:: python
SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"]
.. setting:: STATS_CLASS
@ -1581,66 +1660,77 @@ If a reactor is already installed,
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
In order to use the reactor installed by Scrapy::
In order to use the reactor installed by Scrapy:
.. code-block:: python
import scrapy
from twisted.internet import reactor
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
self.timeout = int(kwargs.pop("timeout", "60"))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['https://quotes.toscrape.com/page/1']
urls = ["https://quotes.toscrape.com/page/1"]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
for quote in response.css("div.quote"):
yield {"text": quote.css("span.text::text").get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
self.crawler.engine.close_spider(self, "timeout")
which raises :exc:`Exception`, becomes::
which raises :exc:`Exception`, becomes:
.. code-block:: python
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
self.timeout = int(kwargs.pop("timeout", "60"))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['https://quotes.toscrape.com/page/1']
urls = ["https://quotes.toscrape.com/page/1"]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
for quote in response.css("div.quote"):
yield {"text": quote.css("span.text::text").get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
self.crawler.engine.close_spider(self, "timeout")
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will install the default reactor defined by Twisted for the
current platform. This is to maintain backward compatibility and avoid possible
problems caused by using a non-default reactor.
means that Scrapy will use the existing reactor if one is already installed, or
install the default reactor defined by Twisted for the current platform. This
is to maintain backward compatibility and avoid possible problems caused by
using a non-default reactor.
.. versionchanged:: 2.7
The :command:`startproject` command now sets this setting to
``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
``settings.py`` file.
For additional information, see :doc:`core/howto/choosing-reactor`.
@ -1656,14 +1746,14 @@ Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs.
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
:setting:`DEPTH_LIMIT`.
Use ``0`` to allow URLs of any length.
The default value is copied from the `Microsoft Internet Explorer maximum URL
The default value is copied from the `Microsoft Internet Explorer maximum URL
length`_, even though this setting exists for different reasons.
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246

81
docs/topics/shell.rst
View File

@ -191,44 +191,46 @@ all start with the ``[s]`` prefix)::
After that, we can start playing with the objects:
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
.. code-block:: pycon
>>> fetch("https://old.reddit.com/")
>>> response.xpath("//title/text()").get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'
>>> fetch("https://old.reddit.com/")
>>> request = request.replace(method="POST")
>>> response.xpath("//title/text()").get()
'reddit: the front page of the internet'
>>> fetch(request)
>>> request = request.replace(method="POST")
>>> response.status
404
>>> fetch(request)
>>> from pprint import pprint
>>> response.status
404
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
>>> from pprint import pprint
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
.. _topics-shell-inspect-response:
@ -242,7 +244,9 @@ getting there.
This can be achieved by using the ``scrapy.shell.inspect_response`` function.
Here's an example of how you would call it from your spider::
Here's an example of how you would call it from your spider:
.. code-block:: python
import scrapy
@ -259,6 +263,7 @@ Here's an example of how you would call it from your spider::
# We want to inspect one specific response.
if ".org" in response.url:
from scrapy.shell import inspect_response
inspect_response(response, self)
# Rest of parsing code.
@ -276,14 +281,18 @@ When you run the spider, you will get something similar to this::
Then, you can check if the extraction code is working:
>>> response.xpath('//h1[@class="fn"]')
[]
.. code-block:: pycon
>>> response.xpath('//h1[@class="fn"]')
[]
Nope, it doesn't. So you can open the response in your web browser and see if
it's the response you were expecting:
>>> view(response)
True
.. code-block:: pycon
>>> view(response)
True
Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
crawling::

79
docs/topics/signals.rst
View File

@ -16,7 +16,9 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action::
Here is a simple example showing how you can catch signals and perform some action:
.. code-block:: python
from scrapy import signals
from scrapy import Spider
@ -30,17 +32,14 @@ Here is a simple example showing how you can catch signals and perform some acti
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def spider_closed(self, spider):
spider.logger.info('Spider closed: %s', spider.name)
spider.logger.info("Spider closed: %s", spider.name)
def parse(self, response):
pass
@ -51,16 +50,21 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
or :term:`awaitable objects <awaitable>` from their handlers, allowing
you to run asynchronous code that does not block Scrapy. If a signal
handler returns one of these objects, Scrapy waits for that asynchronous
operation to finish.
Let's take an example using :ref:`coroutines <topics-coroutines>`:
.. code-block:: python
import scrapy
Let's take an example::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['https://quotes.toscrape.com/page/1/']
name = "signals"
start_urls = ["https://quotes.toscrape.com/page/1/"]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
@ -68,28 +72,26 @@ Let's take an example::
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
async def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
response = await treq.post(
"http://example.com/post",
json.dumps(item).encode("ascii"),
headers={b"Content-Type": [b"application/json"]},
)
# The next item will be scraped only after
# deferred (d) is fired
return d
return response
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
.. _topics-signals-ref:
@ -305,6 +307,33 @@ spider_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object
feed_slot_closed
~~~~~~~~~~~~~~~~
.. signal:: feed_slot_closed
.. function:: feed_slot_closed(slot)
Sent when a :ref:`feed exports <topics-feed-exports>` slot is closed.
This signal supports returning deferreds from its handlers.
:param slot: the slot closed
:type slot: scrapy.extensions.feedexport.FeedSlot
feed_exporter_closed
~~~~~~~~~~~~~~~~~~~~
.. signal:: feed_exporter_closed
.. function:: feed_exporter_closed()
Sent when the :ref:`feed exports <topics-feed-exports>` extension is closed,
during the handling of the :signal:`spider_closed` signal by the extension,
after all feed exporting has been handled.
This signal supports returning deferreds from its handlers.
Request signals
---------------

47
docs/topics/spider-middleware.rst
View File

@ -18,10 +18,12 @@ To activate a spider middleware component, add it to the
:setting:`SPIDER_MIDDLEWARES` setting, which is a dict whose keys are the
middleware class path and their values are the middleware orders.
Here's an example::
Here's an example:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'myproject.middlewares.CustomSpiderMiddleware': 543,
"myproject.middlewares.CustomSpiderMiddleware": 543,
}
The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
@ -44,11 +46,13 @@ previous (or subsequent) middleware being applied.
If you want to disable a builtin middleware (the ones defined in
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
value. For example, if you want to disable the off-site middleware::
value. For example, if you want to disable the off-site middleware:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'myproject.middlewares.CustomSpiderMiddleware': 543,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
"myproject.middlewares.CustomSpiderMiddleware": 543,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -102,27 +106,47 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.Request` objects and :ref:`item object
:class:`~scrapy.Request` objects and :ref:`item objects
<topics-items>`.
.. versionchanged:: 2.7
This method may be defined as an :term:`asynchronous generator`, in
which case ``result`` is an :term:`asynchronous iterable`.
Consider defining this method as an :term:`asynchronous generator`,
which will be a requirement in a future version of Scrapy. However, if
you plan on sharing your spider middleware with other people, consider
either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
as a minimum requirement of your spider middleware, or :ref:`making
your spider middleware universal <universal-spider-middleware>` so that
it works with Scrapy versions earlier than Scrapy 2.7.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item object <topics-items>`
:ref:`item objects <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output_async(response, result, spider)
.. versionadded:: 2.7
If defined, this method must be an :term:`asynchronous generator`,
which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`.
.. method:: process_spider_exception(response, exception, spider)
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
objects.
If it returns ``None``, Scrapy will continue processing this exception,
@ -241,7 +265,12 @@ specify which response codes the spider is able to handle using the
:setting:`HTTPERROR_ALLOWED_CODES` setting.
For example, if you want your spider to handle 404 responses you can do
this::
this:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class MySpider(CrawlSpider):
handle_httpstatus_list = [404]

262
docs/topics/spiders.rst
View File

@ -99,7 +99,7 @@ scrapy.Spider
.. attribute:: crawler
This attribute is set by the :meth:`from_crawler` class method after
initializating the class, and links to the
initializing the class, and links to the
:class:`~scrapy.crawler.Crawler` object to which this spider instance is
bound.
@ -157,15 +157,24 @@ scrapy.Spider
If you want to change the Requests used to start scraping a domain, this is
the method to override. For example, if you need to start by logging in using
a POST request, you could do::
a POST request, you could do:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def start_requests(self):
return [scrapy.FormRequest("http://www.example.com/login",
formdata={'user': 'john', 'pass': 'secret'},
callback=self.logged_in)]
return [
scrapy.FormRequest(
"http://www.example.com/login",
formdata={"user": "john", "pass": "secret"},
callback=self.logged_in,
)
]
def logged_in(self, response):
# here you would extract links to follow and return Requests for
@ -181,9 +190,10 @@ scrapy.Spider
scraped data and/or more URLs to follow. Other Requests callbacks have
the same requirements as the :class:`Spider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.Request` and/or :ref:`item objects
<topics-items>`.
This method, as well as any other Request callback, must return a
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
<topics-items>`, or ``None``.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`
@ -199,63 +209,72 @@ scrapy.Spider
Called when the spider closes. This method provides a shortcut to
signals.connect() for the :signal:`spider_closed` signal.
Let's see an example::
Let's see an example:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
'http://www.example.com/3.html',
"http://www.example.com/1.html",
"http://www.example.com/2.html",
"http://www.example.com/3.html",
]
def parse(self, response):
self.logger.info('A response from %s just arrived!', response.url)
self.logger.info("A response from %s just arrived!", response.url)
Return multiple Requests and items from a single callback::
Return multiple Requests and items from a single callback:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
'http://www.example.com/3.html',
"http://www.example.com/1.html",
"http://www.example.com/2.html",
"http://www.example.com/3.html",
]
def parse(self, response):
for h3 in response.xpath('//h3').getall():
for h3 in response.xpath("//h3").getall():
yield {"title": h3}
for href in response.xpath('//a/@href').getall():
for href in response.xpath("//a/@href").getall():
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :class:`~scrapy.Item` objects::
to give data more structure you can use :class:`~scrapy.Item` objects:
.. skip: next
.. code-block:: python
import scrapy
from myproject.items import MyItem
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
def start_requests(self):
yield scrapy.Request('http://www.example.com/1.html', self.parse)
yield scrapy.Request('http://www.example.com/2.html', self.parse)
yield scrapy.Request('http://www.example.com/3.html', self.parse)
yield scrapy.Request("http://www.example.com/1.html", self.parse)
yield scrapy.Request("http://www.example.com/2.html", self.parse)
yield scrapy.Request("http://www.example.com/3.html", self.parse)
def parse(self, response):
for h3 in response.xpath('//h3').getall():
for h3 in response.xpath("//h3").getall():
yield MyItem(title=h3)
for href in response.xpath('//a/@href').getall():
for href in response.xpath("//a/@href").getall():
yield scrapy.Request(response.urljoin(href), self.parse)
.. _spiderargs:
@ -273,34 +292,43 @@ Spider arguments are passed through the :command:`crawl` command using the
scrapy crawl myspider -a category=electronics
Spiders can access arguments in their `__init__` methods::
Spiders can access arguments in their `__init__` methods:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = [f'http://www.example.com/categories/{category}']
self.start_urls = [f"http://www.example.com/categories/{category}"]
# ...
The default `__init__` method will take any spider arguments
and copy them to the spider as attributes.
The above example can also be written as follows::
The above example can also be written as follows:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def start_requests(self):
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
yield scrapy.Request(f"http://www.example.com/categories/{self.category}")
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
.. skip: next
.. code-block:: python
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
@ -336,10 +364,13 @@ common scraping cases, like following all links on a site based on certain
rules, crawling from `Sitemaps`_, or parsing an XML/CSV feed.
For the examples used in the following spiders, we'll assume you have a project
with a ``TestItem`` declared in a ``myproject.items`` module::
with a ``TestItem`` declared in a ``myproject.items`` module:
.. code-block:: python
import scrapy
class TestItem(scrapy.Item):
id = scrapy.Field()
name = scrapy.Field()
@ -435,38 +466,46 @@ Crawling rules
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
Let's now take a look at an example CrawlSpider with rules::
Let's now take a look at an example CrawlSpider with rules:
.. code-block:: python
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
class MySpider(CrawlSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com"]
rules = (
# Extract links matching 'category.php' (but not matching 'subsection.php')
# and follow links from them (since no callback means follow=True by default).
Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
Rule(LinkExtractor(allow=(r"category\.php",), deny=(r"subsection\.php",))),
# Extract links matching 'item.php' and parse them with the spider's method parse_item
Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
Rule(LinkExtractor(allow=(r"item\.php",)), callback="parse_item"),
)
def parse_item(self, response):
self.logger.info('Hi, this is an item page! %s', response.url)
self.logger.info("Hi, this is an item page! %s", response.url)
item = scrapy.Item()
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)")
item["name"] = response.xpath('//td[@id="item_name"]/text()').get()
item["description"] = response.xpath(
'//td[@id="item_description"]/text()'
).get()
item["link_text"] = response.meta["link_text"]
url = response.xpath('//td[@id="additional_data"]/@href').get()
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
return response.follow(
url, self.parse_additional_page, cb_kwargs=dict(item=item)
)
def parse_additional_page(self, response, item):
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
item["additional_data"] = response.xpath(
'//p[@id="additional_data"]/text()'
).get()
return item
@ -567,25 +606,31 @@ XMLFeedSpider
XMLFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
These spiders are pretty easy to use, let's have a look at one example::
These spiders are pretty easy to use, let's have a look at one example:
.. skip: next
.. code-block:: python
from scrapy.spiders import XMLFeedSpider
from myproject.items import TestItem
class MySpider(XMLFeedSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.xml']
iterator = 'iternodes' # This is actually unnecessary, since it's the default value
itertag = 'item'
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com/feed.xml"]
iterator = "iternodes" # This is actually unnecessary, since it's the default value
itertag = "item"
def parse_node(self, response, node):
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
self.logger.info(
"Hi, this is a <%s> node!: %s", self.itertag, "".join(node.getall())
)
item = TestItem()
item['id'] = node.xpath('@id').get()
item['name'] = node.xpath('name').get()
item['description'] = node.xpath('description').get()
item["id"] = node.xpath("@id").get()
item["name"] = node.xpath("name").get()
item["description"] = node.xpath("description").get()
return item
Basically what we did up there was to create a spider that downloads a feed from
@ -626,26 +671,30 @@ CSVFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
Let's see an example similar to the previous one, but using a
:class:`CSVFeedSpider`::
:class:`CSVFeedSpider`:
.. skip: next
.. code-block:: python
from scrapy.spiders import CSVFeedSpider
from myproject.items import TestItem
class MySpider(CSVFeedSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.csv']
delimiter = ';'
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com/feed.csv"]
delimiter = ";"
quotechar = "'"
headers = ['id', 'name', 'description']
headers = ["id", "name", "description"]
def parse_row(self, response, row):
self.logger.info('Hi, this is a row!: %r', row)
self.logger.info("Hi, this is a row!: %r", row)
item = TestItem()
item['id'] = row['id']
item['name'] = row['name']
item['description'] = row['description']
item["id"] = row["id"]
item["name"] = row["name"]
item["description"] = row["description"]
return item
@ -727,19 +776,22 @@ SitemapSpider
<lastmod>2005-01-01</lastmod>
</url>
We can define a ``sitemap_filter`` function to filter ``entries`` by date::
We can define a ``sitemap_filter`` function to filter ``entries`` by date:
.. code-block:: python
from datetime import datetime
from scrapy.spiders import SitemapSpider
class FilteredSitemapSpider(SitemapSpider):
name = 'filtered_sitemap_spider'
allowed_domains = ['example.com']
sitemap_urls = ['http://example.com/sitemap.xml']
name = "filtered_sitemap_spider"
allowed_domains = ["example.com"]
sitemap_urls = ["http://example.com/sitemap.xml"]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year >= 2005:
yield entry
@ -764,60 +816,72 @@ SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
Simplest example: process all urls discovered through sitemaps using the
``parse`` callback::
``parse`` callback:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/sitemap.xml']
sitemap_urls = ["http://www.example.com/sitemap.xml"]
def parse(self, response):
pass # ... scrape item here ...
pass # ... scrape item here ...
Process some urls with certain callback and other urls with a different
callback::
callback:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/sitemap.xml']
sitemap_urls = ["http://www.example.com/sitemap.xml"]
sitemap_rules = [
('/product/', 'parse_product'),
('/category/', 'parse_category'),
("/product/", "parse_product"),
("/category/", "parse_category"),
]
def parse_product(self, response):
pass # ... scrape product ...
pass # ... scrape product ...
def parse_category(self, response):
pass # ... scrape category ...
pass # ... scrape category ...
Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps
whose url contains ``/sitemap_shop``::
whose url contains ``/sitemap_shop``:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/robots.txt']
sitemap_urls = ["http://www.example.com/robots.txt"]
sitemap_rules = [
('/shop/', 'parse_shop'),
("/shop/", "parse_shop"),
]
sitemap_follow = ['/sitemap_shops']
sitemap_follow = ["/sitemap_shops"]
def parse_shop(self, response):
pass # ... scrape shop here ...
pass # ... scrape shop here ...
Combine SitemapSpider with other sources of urls::
Combine SitemapSpider with other sources of urls:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/robots.txt']
sitemap_urls = ["http://www.example.com/robots.txt"]
sitemap_rules = [
('/shop/', 'parse_shop'),
("/shop/", "parse_shop"),
]
other_urls = ['http://www.example.com/about']
other_urls = ["http://www.example.com/about"]
def start_requests(self):
requests = list(super(MySpider, self).start_requests())
@ -825,10 +889,10 @@ Combine SitemapSpider with other sources of urls::
return requests
def parse_shop(self, response):
pass # ... scrape shop here ...
pass # ... scrape shop here ...
def parse_other(self, response):
pass # ... scrape other here ...
pass # ... scrape other here ...
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index

41
docs/topics/stats.rst
View File

@ -30,10 +30,11 @@ Common Stats Collector uses
===========================
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
attribute. Here is an example of an extension that access stats:
.. code-block:: python
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats
@ -41,31 +42,43 @@ attribute. Here is an example of an extension that access stats::
def from_crawler(cls, crawler):
return cls(crawler.stats)
Set stat value::
Set stat value:
stats.set_value('hostname', socket.gethostname())
.. code-block:: python
Increment stat value::
stats.set_value("hostname", socket.gethostname())
stats.inc_value('custom_count')
Increment stat value:
Set stat value only if greater than previous::
.. code-block:: python
stats.max_value('max_items_scraped', value)
stats.inc_value("custom_count")
Set stat value only if lower than previous::
Set stat value only if greater than previous:
stats.min_value('min_free_memory_percent', value)
.. code-block:: python
stats.max_value("max_items_scraped", value)
Set stat value only if lower than previous:
.. code-block:: python
stats.min_value("min_free_memory_percent", value)
Get stat value:
>>> stats.get_value('custom_count')
1
.. code-block:: pycon
>>> stats.get_value("custom_count")
1
Get all stats:
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
.. code-block:: pycon
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================

11
docs/topics/webservice.rst
View File

@ -1,11 +0,0 @@
.. _topics-webservice:
===========
Web Service
===========
webservice has been moved into a separate project.
It is hosted at:
https://github.com/scrapy-plugins/scrapy-jsonrpc

18
docs/utils/linkfix.py Executable file → Normal file
View File

@ -13,25 +13,26 @@ Author: dufferzafar
"""
import re
import sys
from pathlib import Path
def main():
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(r"(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))")
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
sys.exit(1)
# For every line, fix the respective file
for line in output_lines:
@ -48,17 +49,14 @@ def main():
else:
# If this is a new file
if newfilename != _filename:
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
Path(_filename).write_text(_contents, encoding="utf-8")
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = Path(_filename).read_text(encoding="utf-8")
_contents = _contents.replace(match.group(3), match.group(4))
else:
@ -66,5 +64,5 @@ def main():
print("Not Understood: " + line)
if __name__ == '__main__':
if __name__ == "__main__":
main()

22
extras/qps-bench-server.py
View File

@ -1,13 +1,13 @@
#!/usr/bin/env python
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from time import time
from twisted.internet import reactor
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
class Root(Resource):
def __init__(self):
Resource.__init__(self)
self.concurrent = 0
@ -26,9 +26,9 @@ class Root(Resource):
delta = now - self.lasttime
# reset stats on high iter-request times caused by client restarts
if delta > 3: # seconds
if delta > 3: # seconds
self._reset_stats()
return ''
return ""
self.tail.appendleft(delta)
self.lasttime = now
@ -37,15 +37,17 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
print(
f"samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}"
)
if 'latency' in request.args:
latency = float(request.args['latency'][0])
if "latency" in request.args:
latency = float(request.args["latency"][0])
reactor.callLater(latency, self._finish, request)
return NOT_DONE_YET
self.concurrent -= 1
return ''
return ""
def _finish(self, request):
self.concurrent -= 1

18
extras/qpsclient.py
View File

@ -1,25 +1,25 @@
"""
A spider that generate light requests to meassure QPS throughput
A spider that generate light requests to measure QPS throughput
usage:
scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0
--set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
"""
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.spiders import Spider
class QPSSpider(Spider):
name = 'qps'
benchurl = 'http://localhost:8880/'
name = "qps"
benchurl = "http://localhost:8880/"
# Max concurrency is limited by global CONCURRENT_REQUESTS setting
max_concurrent_requests = 8
# Requests per second goal
qps = None # same as: 1 / download_delay
qps = None # same as: 1 / download_delay
download_delay = None
# time in seconds to delay server responses
latency = None
@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += f'?latency={self.latency}'
url += f"?latency={self.latency}"
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
urls = [url.replace("localhost", f"127.0.0.{x + 1}") for x in range(slots)]
else:
urls = [url]

37
pylintrc
View File

@ -9,35 +9,27 @@ disable=abstract-method,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-continuation,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bad-whitespace,
bare-except,
blacklisted-name,
broad-except,
broad-exception-raised,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-dict-items,
consider-using-from-import,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
disallowed-name,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
fixme,
function-redefined,
global-statement,
implicit-str-concat,
import-error,
import-outside-toplevel,
import-self,
@ -49,26 +41,20 @@ disable=abstract-method,
keyword-arg-before-vararg,
line-too-long,
logging-format-interpolation,
logging-fstring-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
misplaced-comparison-constant,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-init,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-self-use,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-exception-statement,
pointless-statement,
pointless-string-statement,
protected-access,
@ -78,10 +64,7 @@ disable=abstract-method,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
@ -93,30 +76,24 @@ disable=abstract-method,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-lambda,
unnecessary-dunder-call,
unnecessary-pass,
unreachable,
unspecified-encoding,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
use-dict-literal,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

6
pytest.ini
View File

@ -5,7 +5,6 @@ python_files=test_*.py __init__.py
python_classes=
addopts =
--assert=plain
--doctest-modules
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
@ -21,3 +20,8 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
ignore:typing.re is deprecated
ignore:typing.io is deprecated

2
scrapy/VERSION
View File

@ -1 +1 @@
2.6.1
2.9.0

26
scrapy/__init__.py
View File

@ -9,32 +9,38 @@ import warnings
from twisted import version as _txv
# Declare top-level shortcuts
from scrapy.spiders import Spider
from scrapy.http import Request, FormRequest
from scrapy.http import FormRequest, Request
from scrapy.item import Field, Item
from scrapy.selector import Selector
from scrapy.item import Item, Field
from scrapy.spiders import Spider
__all__ = [
'__version__', 'version_info', 'twisted_version', 'Spider',
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
"__version__",
"version_info",
"twisted_version",
"Spider",
"Request",
"FormRequest",
"Selector",
"Item",
"Field",
]
# Scrapy and Twisted versions
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 6):
print(f"Scrapy {__version__} requires Python 3.6+")
if sys.version_info < (3, 8):
print(f"Scrapy {__version__} requires Python 3.8+")
sys.exit(1)
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
del pkgutil

2
scrapy/__main__.py
View File

@ -1,4 +1,4 @@
from scrapy.cmdline import execute
if __name__ == '__main__':
if __name__ == "__main__":
execute()

58
scrapy/cmdline.py
View File

@ -1,21 +1,30 @@
import sys
import os
import argparse
import cProfile
import inspect
import pkg_resources
import os
import sys
from importlib.metadata import entry_points
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == "-:":
return None
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# TODO: add `name` attribute to commands and merge this function with
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
for obj in vars(module).values():
@ -23,7 +32,7 @@ def _iter_command_classes(module_name):
inspect.isclass(obj)
and issubclass(obj, ScrapyCommand)
and obj.__module__ == module.__name__
and not obj == ScrapyCommand
and obj not in (ScrapyCommand, BaseRunSpiderCommand)
):
yield obj
@ -32,14 +41,14 @@ def _get_commands_from_module(module, inproject):
d = {}
for cmd in _iter_command_classes(module):
if inproject or not cmd.requires_project:
cmdname = cmd.__module__.split('.')[-1]
cmdname = cmd.__module__.split(".")[-1]
d[cmdname] = cmd()
return d
def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
cmds = {}
for entry_point in pkg_resources.iter_entry_points(group):
for entry_point in entry_points().get(group, {}):
obj = entry_point.load()
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
@ -49,9 +58,9 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
def _get_commands_dict(settings, inproject):
cmds = _get_commands_from_module('scrapy.commands', inproject)
cmds = _get_commands_from_module("scrapy.commands", inproject)
cmds.update(_get_commands_from_entry_points(inproject))
cmds_module = settings['COMMANDS_MODULE']
cmds_module = settings["COMMANDS_MODULE"]
if cmds_module:
cmds.update(_get_commands_from_module(cmds_module, inproject))
return cmds
@ -60,7 +69,7 @@ def _get_commands_dict(settings, inproject):
def _pop_command_name(argv):
i = 0
for arg in argv[1:]:
if not arg.startswith('-'):
if not arg.startswith("-"):
del argv[i]
return arg
i += 1
@ -69,7 +78,8 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
else:
print(f"Scrapy {version} - no active project\n")
@ -114,11 +124,11 @@ def execute(argv=None, settings=None):
settings = get_project_settings()
# set EDITOR from environment if available
try:
editor = os.environ['EDITOR']
editor = os.environ["EDITOR"]
except KeyError:
pass
else:
settings['EDITOR'] = editor
settings["EDITOR"] = editor
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
@ -131,11 +141,13 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
parser = ScrapyArgumentParser(
formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler="resolve",
description=cmd.long_desc(),
)
settings.setdict(cmd.default_settings, priority="command")
cmd.settings = settings
cmd.add_options(parser)
opts, args = parser.parse_known_args(args=argv[1:])
@ -158,12 +170,12 @@ def _run_command_profiled(cmd, args, opts):
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)
p.runctx("cmd.run(args, opts)", globals(), loc)
if opts.profile:
p.dump_stats(opts.profile)
if __name__ == '__main__':
if __name__ == "__main__":
try:
execute()
finally:

137
scrapy/commands/__init__.py
View File

@ -1,31 +1,32 @@
"""
Base class for Scrapy commands
"""
import os
import argparse
from typing import Any, Dict
import os
from pathlib import Path
from typing import Any, Dict, Optional
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
class ScrapyCommand:
requires_project = False
crawler_process = None
crawler_process: Optional[CrawlerProcess] = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}
exitcode = 0
def __init__(self):
self.settings = None # set in scrapy.cmdline
def __init__(self) -> None:
self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
if hasattr(self, '_crawler'):
if hasattr(self, "_crawler"):
raise RuntimeError("crawler already set")
self._crawler = crawler
@ -59,42 +60,58 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
group = parser.add_argument_group(title='Global Options')
group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_argument("--nolog", action="store_true",
help="disable logging completely")
group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group = parser.add_argument_group(title="Global Options")
group.add_argument(
"--logfile", metavar="FILE", help="log file. if omitted stderr will be used"
)
group.add_argument(
"-L",
"--loglevel",
metavar="LEVEL",
default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})",
)
group.add_argument(
"--nolog", action="store_true", help="disable logging completely"
)
group.add_argument(
"--profile",
metavar="FILE",
default=None,
help="write python cProfile stats to FILE",
)
group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE")
group.add_argument(
"-s",
"--set",
action="append",
default=[],
metavar="NAME=VALUE",
help="set/override setting (may be repeated)",
)
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
self.settings.setdict(arglist_to_dict(opts.set),
priority='cmdline')
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
if opts.logfile:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
self.settings.set("LOG_ENABLED", True, priority="cmdline")
self.settings.set("LOG_FILE", opts.logfile, priority="cmdline")
if opts.loglevel:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
self.settings.set("LOG_ENABLED", True, priority="cmdline")
self.settings.set("LOG_LEVEL", opts.loglevel, priority="cmdline")
if opts.nolog:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
self.settings.set("LOG_ENABLED", False, priority="cmdline")
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
Path(opts.pidfile).write_text(
str(os.getpid()) + os.linesep, encoding="utf-8"
)
if opts.pdb:
failure.startDebugMode()
@ -110,16 +127,39 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
parser.add_argument(
"-a",
dest="spargs",
action="append",
default=[],
metavar="NAME=VALUE",
help="set spider argument (may be repeated)",
)
parser.add_argument(
"-o",
"--output",
metavar="FILE",
action="append",
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)",
)
parser.add_argument(
"-O",
"--overwrite-output",
metavar="FILE",
action="append",
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
)
parser.add_argument(
"-t",
"--output-format",
metavar="FORMAT",
help="format to use for dumping items",
)
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -134,16 +174,21 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
self.settings.set("FEEDS", feeds, priority="cmdline")
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
super().__init__(
prog,
indent_increment=indent_increment,
max_help_position=max_help_position,
width=width,
)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
@ -154,11 +199,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
part_strings[0] = "Usage\n=====\n " + part_strings[0][len("usage: ") :]
headings = [
i for i in range(len(part_strings)) if part_strings[i].endswith(":\n")
]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
char = "-" if "Global Options" in part_strings[index] else "="
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
underline = "".join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

25
scrapy/commands/bench.py
View File

@ -1,6 +1,6 @@
import subprocess
import sys
import time
import subprocess
from urllib.parse import urlencode
import scrapy
@ -9,11 +9,10 @@ from scrapy.linkextractors import LinkExtractor
class Command(ScrapyCommand):
default_settings = {
'LOG_LEVEL': 'INFO',
'LOGSTATS_INTERVAL': 1,
'CLOSESPIDER_TIMEOUT': 10,
"LOG_LEVEL": "INFO",
"LOGSTATS_INTERVAL": 1,
"CLOSESPIDER_TIMEOUT": 10,
}
def short_desc(self):
@ -26,12 +25,11 @@ class Command(ScrapyCommand):
class _BenchServer:
def __enter__(self):
from scrapy.utils.test import get_testenv
pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver']
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE,
env=get_testenv())
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
self.proc.stdout.readline()
def __exit__(self, exc_type, exc_value, traceback):
@ -42,15 +40,16 @@ class _BenchServer:
class _BenchSpider(scrapy.Spider):
"""A spider that follows all links"""
name = 'follow'
name = "follow"
total = 10000
show = 20
baseurl = 'http://localhost:8998'
baseurl = "http://localhost:8998"
link_extractor = LinkExtractor()
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
qargs = {"total": self.total, "show": self.show}
url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

32
scrapy/commands/check.py
View File

@ -1,11 +1,12 @@
import time
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
from unittest import TextTestResult as _TextTestResult
from unittest import TextTestRunner
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object, set_environ
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):
@ -39,7 +40,7 @@ class TextTestResult(_TextTestResult):
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "[options] <spider>"
@ -49,14 +50,25 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
parser.add_argument(
"-l",
"--list",
dest="list",
action="store_true",
help="only list contracts, without checking them",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
default=False,
action="store_true",
help="print contract tests for all spiders",
)
def run(self, args, opts):
# load contracts
contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
conman = ContractsManager(load_object(c) for c in contracts)
runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
@ -66,7 +78,7 @@ class Command(ScrapyCommand):
spider_loader = self.crawler_process.spider_loader
with set_environ(SCRAPY_CHECK='true'):
with set_environ(SCRAPY_CHECK="true"):
for spidername in args or spider_loader.list():
spidercls = spider_loader.load(spidername)
spidercls.start_requests = lambda s: conman.from_spider(s, result)
@ -85,7 +97,7 @@ class Command(ScrapyCommand):
continue
print(spider)
for method in sorted(methods):
print(f' * {method}')
print(f" * {method}")
else:
start = time.time()
self.crawler_process.start()

12
scrapy/commands/crawl.py
View File

@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError
class Command(BaseRunSpiderCommand):
requires_project = True
def syntax(self):
@ -16,18 +15,23 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
raise UsageError(
"running 'scrapy crawl' with more than one spider is not supported"
)
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
if getattr(crawl_defer, "result", None) is not None and issubclass(
crawl_defer.result.type, Exception
):
self.exitcode = 1
else:
self.crawler_process.start()
if (
self.crawler_process.bootstrap_failed
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
or hasattr(self.crawler_process, "has_exception")
and self.crawler_process.has_exception
):
self.exitcode = 1

15
scrapy/commands/edit.py
View File

@ -1,14 +1,13 @@
import sys
import os
import sys
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "<spider>"
@ -17,8 +16,10 @@ class Command(ScrapyCommand):
return "Edit spider"
def long_desc(self):
return ("Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting")
return (
"Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting"
)
def _err(self, msg):
sys.stderr.write(msg + os.linesep)
@ -28,12 +29,12 @@ class Command(ScrapyCommand):
if len(args) != 1:
raise UsageError()
editor = self.settings['EDITOR']
editor = self.settings["EDITOR"]
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
sfile = sfile.replace(".pyc", ".py")
self.exitcode = os.system(f'{editor} "{sfile}"')

45
scrapy/commands/fetch.py
View File

@ -1,15 +1,15 @@
import sys
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
class Command(ScrapyCommand):
requires_project = False
def syntax(self):
@ -27,38 +27,51 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument(
"--headers",
dest="headers",
action="store_true",
help="print response HTTP headers instead of body",
)
parser.add_argument(
"--no-redirect",
dest="no_redirect",
action="store_true",
default=False,
help="do not handle HTTP 3xx status codes and print response as-is",
)
def _print_headers(self, headers, prefix):
for key, values in headers.items():
for value in values:
self._print_bytes(prefix + b' ' + key + b': ' + value)
self._print_bytes(prefix + b" " + key + b": " + value)
def _print_response(self, response, opts):
if opts.headers:
self._print_headers(response.request.headers, b'>')
print('>')
self._print_headers(response.headers, b'<')
self._print_headers(response.request.headers, b">")
print(">")
self._print_headers(response.headers, b"<")
else:
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
sys.stdout.buffer.write(bytes_ + b'\n')
sys.stdout.buffer.write(bytes_ + b"\n")
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
request = Request(
args[0],
callback=self._print_response,
cb_kwargs={"opts": opts},
dont_filter=True,
)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:
request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
request.meta["handle_httpstatus_list"] = SequenceExclude(range(300, 400))
else:
request.meta['handle_httpstatus_all'] = True
request.meta["handle_httpstatus_all"] = True
spidercls = DefaultSpider
spider_loader = self.crawler_process.spider_loader

155
scrapy/commands/genspider.py
View File

@ -1,15 +1,15 @@
import os
import shutil
import string
from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext
from pathlib import Path
from typing import Optional, cast
from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
def sanitize_module_name(module_name):
@ -17,7 +17,7 @@ def sanitize_module_name(module_name):
with underscores and prefixing it with a letter if it doesn't start
with one
"""
module_name = module_name.replace('-', '_').replace('.', '_')
module_name = module_name.replace("-", "_").replace(".", "_")
if module_name[0] not in string.ascii_letters:
module_name = "a" + module_name
return module_name
@ -26,15 +26,22 @@ def sanitize_module_name(module_name):
def extract_domain(url):
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == '' and o.netloc == '':
if o.scheme == "" and o.netloc == "":
o = urlparse("//" + url.lstrip("/"))
return o.netloc
class Command(ScrapyCommand):
def verify_url_scheme(url):
"""Check url for scheme and insert https if none found."""
parsed = urlparse(url)
if parsed.scheme == "" and parsed.netloc == "":
parsed = urlparse("//" + url)._replace(scheme="https")
return parsed.geturl()
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def syntax(self):
return "[options] <name> <domain>"
@ -44,16 +51,40 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
parser.add_argument(
"-l",
"--list",
dest="list",
action="store_true",
help="List available templates",
)
parser.add_argument(
"-e",
"--edit",
dest="edit",
action="store_true",
help="Edit spider after creating it",
)
parser.add_argument(
"-d",
"--dump",
dest="dump",
metavar="TEMPLATE",
help="Dump template to standard output",
)
parser.add_argument(
"-t",
"--template",
dest="template",
default="basic",
help="Uses a custom template.",
)
parser.add_argument(
"--force",
dest="force",
action="store_true",
help="If the spider already exists, overwrite it with the template",
)
def run(self, args, opts):
if opts.list:
@ -62,17 +93,16 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
with open(template_file, "r") as f:
print(f.read())
print(template_file.read_text(encoding="utf-8"))
return
if len(args) != 2:
raise UsageError()
name, url = args[0:2]
domain = extract_domain(url)
url = verify_url_scheme(url)
module = sanitize_module_name(name)
if self.settings.get('BOT_NAME') == module:
if self.settings.get("BOT_NAME") == module:
print("Cannot create a spider with the same name as your project")
return
@ -81,56 +111,66 @@ class Command(ScrapyCommand):
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
def _genspider(self, module, name, url, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
domain = extract_domain(url)
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': f'{capitalized_module}Spider'
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
"url": url,
"domain": domain,
"classname": f"{capitalized_module}Spider",
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = abspath(dirname(spiders_module.__file__))
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
spiders_dir = "."
spider_file = f"{join(spiders_dir, module)}.py"
spiders_dir = Path(".")
spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
print(
f"Created spider {name!r} using template {template_name!r} ",
end=("" if spiders_module else "\n"),
)
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
def _find_template(self, template: str) -> Optional[Path]:
template_file = Path(self.templates_dir, f"{template}.tmpl")
if template_file.exists():
return template_file
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
return None
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(f" {splitext(filename)[0]}")
for file in sorted(Path(self.templates_dir).iterdir()):
if file.suffix == ".tmpl":
print(f" {file.stem}")
def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'):
def _spider_exists(self, name: str) -> bool:
if not self.settings.get("NEWSPIDER_MODULE"):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
path = Path(name + ".py")
if path.exists():
print(f"{path.resolve()} already exists")
return True
return False
assert (
self.crawler_process is not None
), "crawler_process must be set before calling run"
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
@ -142,18 +182,21 @@ class Command(ScrapyCommand):
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
spiders_dir = Path(cast(str, spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():
print(f"{path} already exists")
return True
return False
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'spiders'
def templates_dir(self) -> str:
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
"spiders",
)
)

3
scrapy/commands/list.py
View File

@ -2,9 +2,8 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
default_settings = {"LOG_ENABLED": False}
def short_desc(self):
return "List available spiders"

237
scrapy/commands/parse.py
View File

@ -1,16 +1,21 @@
import inspect
import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import maybeDeferred
from w3lib.url import is_url
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
logger = logging.getLogger(__name__)
@ -32,28 +37,72 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
parser.add_argument(
"--spider",
dest="spider",
default=None,
help="use this spider without looking for one",
)
parser.add_argument(
"--pipelines", action="store_true", help="process items through pipelines"
)
parser.add_argument(
"--nolinks",
dest="nolinks",
action="store_true",
help="don't show links to follow (extracted requests)",
)
parser.add_argument(
"--noitems",
dest="noitems",
action="store_true",
help="don't show scraped items",
)
parser.add_argument(
"--nocolour",
dest="nocolour",
action="store_true",
help="avoid using pygments to colorize the output",
)
parser.add_argument(
"-r",
"--rules",
dest="rules",
action="store_true",
help="use CrawlSpider rules to discover the callback",
)
parser.add_argument(
"-c",
"--callback",
dest="callback",
help="use this callback for parsing, instead looking for a callback",
)
parser.add_argument(
"-m",
"--meta",
dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string",
)
parser.add_argument(
"--cbkwargs",
dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string",
)
parser.add_argument(
"-d",
"--depth",
dest="depth",
type=int,
default=1,
help="maximum depth for parsing requests [default: %(default)s]",
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
action="store_true",
help="print each depth level one by one",
)
@property
def max_level(self):
@ -64,6 +113,25 @@ class Command(BaseRunSpiderCommand):
max_requests = max(self.requests)
return max(max_items, max_requests)
def handle_exception(self, _failure):
logger.error(
"An error is caught while iterating the async iterable",
exc_info=failure_to_exc_info(_failure),
)
def iterate_spider_output(self, result):
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))
)
d.addCallback(self.iterate_spider_output)
return d
if inspect.iscoroutine(result):
d = deferred_from_coro(result)
d.addCallback(self.iterate_spider_output)
return d
return arg_to_iter(deferred_from_coro(result))
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
self.items[lvl] = old_items + new_items
@ -98,38 +166,43 @@ class Command(BaseRunSpiderCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print(f'\n>>> DEPTH LEVEL: {level} <<<')
print(f"\n>>> DEPTH LEVEL: {level} <<<")
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
print(f"\n>>> STATUS DEPTH LEVEL {self.max_level} <<<")
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:
self.print_requests(colour=colour)
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
items, requests = [], []
for x in iterate_spider_output(callback(response, **cb_kwargs)):
for x in spider_output:
if is_item(x):
items.append(x)
elif isinstance(x, Request):
requests.append(x)
return items, requests
return items, requests, opts, depth, spider, callback
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(self, spider, response):
if getattr(spider, 'rules', None):
if getattr(spider, "rules", None):
for rule in spider.rules:
if rule.link_extractor.matches(response.url):
return rule.callback or "parse"
else:
logger.error('No CrawlSpider rules found in spider %(spider)r, '
'please specify a callback to use for parsing',
{'spider': spider.name})
logger.error(
"No CrawlSpider rules found in spider %(spider)r, "
"please specify a callback to use for parsing",
{"spider": spider.name},
)
def set_spidercls(self, url, opts):
spider_loader = self.crawler_process.spider_loader
@ -137,16 +210,19 @@ class Command(BaseRunSpiderCommand):
try:
self.spidercls = spider_loader.load(opts.spider)
except KeyError:
logger.error('Unable to find spider: %(spider)s',
{'spider': opts.spider})
logger.error(
"Unable to find spider: %(spider)s", {"spider": opts.spider}
)
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
logger.error("Unable to find spider for: %(url)s", {"url": url})
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
if self.spidercls:
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
self.crawler_process.crawl(self.spidercls, **opts.spargs)
@ -154,8 +230,26 @@ class Command(BaseRunSpiderCommand):
self.crawler_process.start()
if not self.first_response:
logger.error('No response downloaded for: %(url)s',
{'url': url})
logger.error("No response downloaded for: %(url)s", {"url": url})
def scraped_data(self, args):
items, requests, opts, depth, spider, callback = args
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, spider)
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta["_depth"] = depth + 1
req.meta["_callback"] = req.callback
req.callback = callback
scraped_data += requests
return scraped_data
def prepare_request(self, spider, request, opts):
def callback(response, **cb_kwargs):
@ -164,7 +258,7 @@ class Command(BaseRunSpiderCommand):
self.first_response = response
# determine real callback
cb = response.meta['_callback']
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
@ -172,41 +266,32 @@ class Command(BaseRunSpiderCommand):
cb = self.get_callback_from_rules(spider, response)
if not cb:
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
{'url': response.url, 'spider': spider.name})
logger.error(
"Cannot find a rule that matches %(url)r in spider: %(spider)s",
{"url": response.url, "spider": spider.name},
)
return
else:
cb = 'parse'
cb = "parse"
if not callable(cb):
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
{'callback': cb, 'spider': spider.name})
logger.error(
"Cannot find callback %(callback)r in spider: %(spider)s",
{"callback": cb, "spider": spider.name},
)
return
# parse items and requests
depth = response.meta['_depth']
depth = response.meta["_depth"]
items, requests = self.run_callback(response, cb, cb_kwargs)
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, spider)
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
scraped_data += requests
return scraped_data
d = self.run_callback(response, cb, cb_kwargs)
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
d.addCallback(self.scraped_data)
return d
# update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
@ -216,8 +301,8 @@ class Command(BaseRunSpiderCommand):
if opts.cbkwargs:
request.cb_kwargs.update(opts.cbkwargs)
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.meta["_depth"] = 1
request.meta["_callback"] = request.callback
request.callback = callback
return request
@ -232,16 +317,22 @@ class Command(BaseRunSpiderCommand):
try:
opts.meta = json.loads(opts.meta)
except ValueError:
raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
"Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
raise UsageError(
"Invalid -m/--meta value, pass a valid json string to -m or --meta. "
'Example: --meta=\'{"foo" : "bar"}\'',
print_help=False,
)
def process_request_cb_kwargs(self, opts):
if opts.cbkwargs:
try:
opts.cbkwargs = json.loads(opts.cbkwargs)
except ValueError:
raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
"Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
raise UsageError(
"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
'Example: --cbkwargs=\'{"foo" : "bar"}\'',
print_help=False,
)
def run(self, args, opts):
# parse arguments

35
scrapy/commands/runspider.py
View File

@ -1,32 +1,31 @@
import sys
import os
from importlib import import_module
from os import PathLike
from pathlib import Path
from types import ModuleType
from typing import Union
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext not in ('.py', '.pyw'):
def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in (".py", ".pyw"):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
dirname = str(abspath.parent)
sys.path = [dirname] + sys.path
try:
module = import_module(fname)
module = import_module(abspath.stem)
finally:
if dirname:
sys.path.pop(0)
sys.path.pop(0)
return module
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[options] <spider_file>"
@ -40,13 +39,13 @@ class Command(BaseRunSpiderCommand):
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
filename = Path(args[0])
if not filename.exists():
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError(f"Unable to load {filename!r}: {e}\n")
raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n")

41
scrapy/commands/settings.py
View File

@ -5,10 +5,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[options]"
@ -18,16 +16,33 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
parser.add_argument(
"--get", dest="get", metavar="SETTING", help="print raw setting value"
)
parser.add_argument(
"--getbool",
dest="getbool",
metavar="SETTING",
help="print setting value, interpreted as a boolean",
)
parser.add_argument(
"--getint",
dest="getint",
metavar="SETTING",
help="print setting value, interpreted as an integer",
)
parser.add_argument(
"--getfloat",
dest="getfloat",
metavar="SETTING",
help="print setting value, interpreted as a float",
)
parser.add_argument(
"--getlist",
dest="getlist",
metavar="SETTING",
help="print setting value, interpreted as a list",
)
def run(self, args, opts):
settings = self.crawler_process.settings

45
scrapy/commands/shell.py
View File

@ -8,17 +8,16 @@ from threading import Thread
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.shell import Shell
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
class Command(ScrapyCommand):
requires_project = False
default_settings = {
'KEEP_ALIVE': True,
'LOGSTATS_INTERVAL': 0,
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
"KEEP_ALIVE": True,
"LOGSTATS_INTERVAL": 0,
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
}
def syntax(self):
@ -28,17 +27,26 @@ class Command(ScrapyCommand):
return "Interactive scraping console"
def long_desc(self):
return ("Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file.")
return (
"Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file."
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_argument("--spider", dest="spider",
help="use this spider")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument(
"-c",
dest="code",
help="evaluate the code in the shell, print the result and exit",
)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument(
"--no-redirect",
dest="no_redirect",
action="store_true",
default=False,
help="do not handle HTTP 3xx status codes and print response as-is",
)
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
@ -58,8 +66,9 @@ class Command(ScrapyCommand):
if opts.spider:
spidercls = spider_loader.load(opts.spider)
elif url:
spidercls = spidercls_for_request(spider_loader, Request(url),
spidercls, log_multiple=True)
spidercls = spidercls_for_request(
spider_loader, Request(url), spidercls, log_multiple=True
)
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
@ -74,7 +83,9 @@ class Command(ScrapyCommand):
shell.start(url=url, redirect=not opts.no_redirect)
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
t = Thread(
target=self.crawler_process.start,
kwargs={"stop_after_crawl": False, "install_signal_handlers": False},
)
t.daemon = True
t.start()

95
scrapy/commands/startproject.py
View File

@ -1,26 +1,25 @@
import re
import os
import re
import string
from importlib.util import find_spec
from os.path import join, exists, abspath
from shutil import ignore_patterns, move, copy2, copystat
from pathlib import Path
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
TEMPLATES_TO_RENDER = (
('scrapy.cfg',),
('${project_name}', 'settings.py.tmpl'),
('${project_name}', 'items.py.tmpl'),
('${project_name}', 'pipelines.py.tmpl'),
('${project_name}', 'middlewares.py.tmpl'),
("scrapy.cfg",),
("${project_name}", "settings.py.tmpl"),
("${project_name}", "items.py.tmpl"),
("${project_name}", "pipelines.py.tmpl"),
("${project_name}", "middlewares.py.tmpl"),
)
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
def _make_writable(path):
@ -29,10 +28,8 @@ def _make_writable(path):
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "<project_name> [project_dir]"
@ -45,16 +42,18 @@ class Command(ScrapyCommand):
spec = find_spec(module_name)
return spec is not None and spec.loader is not None
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
if not re.search(r"^[_a-zA-Z]\w*$", project_name):
print(
"Error: Project names must begin with a letter and contain"
" only\nletters, numbers and underscores"
)
elif _module_exists(project_name):
print(f'Error: Module {project_name!r} already exists')
print(f"Error: Module {project_name!r} already exists")
else:
return True
return False
def _copytree(self, src, dst):
def _copytree(self, src: Path, dst: Path):
"""
Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and
@ -64,19 +63,19 @@ class Command(ScrapyCommand):
https://github.com/scrapy/scrapy/pull/2005
"""
ignore = IGNORE
names = os.listdir(src)
names = [x.name for x in src.iterdir()]
ignored_names = ignore(src, names)
if not os.path.exists(dst):
os.makedirs(dst)
if not dst.exists():
dst.mkdir(parents=True)
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if os.path.isdir(srcname):
srcname = src / name
dstname = dst / name
if srcname.is_dir():
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
@ -90,36 +89,50 @@ class Command(ScrapyCommand):
raise UsageError()
project_name = args[0]
project_dir = args[0]
if len(args) == 2:
project_dir = args[1]
project_dir = Path(args[1])
else:
project_dir = Path(args[0])
if exists(join(project_dir, 'scrapy.cfg')):
if (project_dir / "scrapy.cfg").exists():
self.exitcode = 1
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}")
return
if not self._is_valid_name(project_name):
self.exitcode = 1
return
self._copytree(self.templates_dir, abspath(project_dir))
move(join(project_dir, 'module'), join(project_dir, project_name))
self._copytree(Path(self.templates_dir), project_dir.resolve())
move(project_dir / "module", project_dir / project_name)
for paths in TEMPLATES_TO_RENDER:
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
tplfile = Path(
project_dir,
*(
string.Template(s).substitute(project_name=project_name)
for s in paths
),
)
render_templatefile(
tplfile,
project_name=project_name,
ProjectName=string_camelcase(project_name),
)
print(
f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:"
)
print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:")
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'project'
def templates_dir(self) -> str:
return str(
Path(
self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"),
"project",
)
)

13
scrapy/commands/version.py
View File

@ -4,9 +4,7 @@ from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
return "[-v]"
@ -16,8 +14,13 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store_true",
help="also display twisted/python/platform info (useful for bug reports)",
)
def run(self, args, opts):
if opts.verbose:

8
scrapy/commands/view.py
View File

@ -1,19 +1,21 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
class Command(fetch.Command):
def short_desc(self):
return "Open URL in browser, as seen by Scrapy"
def long_desc(self):
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
return (
"Fetch a URL using the Scrapy downloader and show its contents in a browser"
)
def add_options(self, parser):
super().add_options(parser)
parser.add_argument('--headers', help=argparse.SUPPRESS)
parser.add_argument("--headers", help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)

35
scrapy/contracts/__init__.py
View File

@ -11,16 +11,17 @@ from scrapy.utils.spider import iterate_spider_output
class Contract:
""" Abstract class for contracts """
"""Abstract class for contracts"""
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
if hasattr(self, "pre_process"):
cb = request.callback
@wraps(cb)
@ -43,7 +44,7 @@ class Contract:
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
if hasattr(self, "post_process"):
cb = request.callback
@wraps(cb)
@ -88,12 +89,12 @@ class ContractsManager:
def extract_contracts(self, method):
contracts = []
for line in method.__doc__.split('\n'):
for line in method.__doc__.split("\n"):
line = line.strip()
if line.startswith('@'):
name, args = re.match(r'@(\w+)\s*(.*)', line).groups()
args = re.split(r'\s+', args)
if line.startswith("@"):
name, args = re.match(r"@(\w+)\s*(.*)", line).groups()
args = re.split(r"\s+", args)
contracts.append(self.contracts[name](method, *args))
@ -106,7 +107,7 @@ class ContractsManager:
try:
requests.append(self.from_method(bound_method, results))
except Exception:
case = _create_testcase(bound_method, 'contract')
case = _create_testcase(bound_method, "contract")
results.addError(case, sys.exc_info())
return requests
@ -124,13 +125,13 @@ class ContractsManager:
# Don't filter requests to allow
# testing different callbacks on the same URL.
kwargs['dont_filter'] = True
kwargs['callback'] = method
kwargs["dont_filter"] = True
kwargs["callback"] = method
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
args.remove('self')
args.remove("self")
# check if all positional arguments are defined in kwargs
if set(args).issubset(set(kwargs)):
@ -146,7 +147,7 @@ class ContractsManager:
return request
def _clean_req(self, request, method, results):
""" stop the request from returning objects and records any errors """
"""stop the request from returning objects and records any errors"""
cb = request.callback
@ -156,11 +157,11 @@ class ContractsManager:
output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
except Exception:
case = _create_testcase(method, 'callback')
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())
def eb_wrapper(failure):
case = _create_testcase(method, 'errback')
case = _create_testcase(method, "errback")
exc_info = failure.type, failure.value, failure.getTracebackObject()
results.addError(case, exc_info)
@ -175,6 +176,6 @@ def _create_testcase(method, desc):
def __str__(_self):
return f"[{spider}] {method.__name__} ({desc})"
name = f'{spider}_{method.__name__}'
name = f"{spider}_{method.__name__}"
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

62
scrapy/contracts/default.py
View File

@ -1,6 +1,6 @@
import json
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
@ -9,50 +9,50 @@ from scrapy.http import Request
# contracts
class UrlContract(Contract):
""" Contract to set the url of the request (mandatory)
@url http://scrapy.org
"""Contract to set the url of the request (mandatory)
@url http://scrapy.org
"""
name = 'url'
name = "url"
def adjust_request_args(self, args):
args['url'] = self.args[0]
args["url"] = self.args[0]
return args
class CallbackKeywordArgumentsContract(Contract):
""" Contract to set the keyword arguments for the request.
The value should be a JSON-encoded dictionary, e.g.:
"""Contract to set the keyword arguments for the request.
The value should be a JSON-encoded dictionary, e.g.:
@cb_kwargs {"arg1": "some value"}
@cb_kwargs {"arg1": "some value"}
"""
name = 'cb_kwargs'
name = "cb_kwargs"
def adjust_request_args(self, args):
args['cb_kwargs'] = json.loads(' '.join(self.args))
args["cb_kwargs"] = json.loads(" ".join(self.args))
return args
class ReturnsContract(Contract):
""" Contract to check the output of a callback
"""Contract to check the output of a callback
general form:
@returns request(s)/item(s) [min=1 [max]]
general form:
@returns request(s)/item(s) [min=1 [max]]
e.g.:
@returns request
@returns request 2
@returns request 2 10
@returns request 0 10
e.g.:
@returns request
@returns request 2
@returns request 2 10
@returns request 0 10
"""
name = 'returns'
name = "returns"
object_type_verifiers = {
'request': lambda x: isinstance(x, Request),
'requests': lambda x: isinstance(x, Request),
'item': is_item,
'items': is_item,
"request": lambda x: isinstance(x, Request),
"requests": lambda x: isinstance(x, Request),
"item": is_item,
"items": is_item,
}
def __init__(self, *args, **kwargs):
@ -73,7 +73,7 @@ class ReturnsContract(Contract):
try:
self.max_bound = int(self.args[2])
except IndexError:
self.max_bound = float('inf')
self.max_bound = float("inf")
def post_process(self, output):
occurrences = 0
@ -81,23 +81,25 @@ class ReturnsContract(Contract):
if self.obj_type_verifier(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
assertion = self.min_bound <= occurrences <= self.max_bound
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = f'{self.min_bound}..{self.max_bound}'
expected = f"{self.min_bound}..{self.max_bound}"
raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
raise ContractFail(
f"Returned {occurrences} {self.obj_name}, expected {expected}"
)
class ScrapesContract(Contract):
""" Contract to check presence of fields in scraped items
@scrapes page_name page_body
"""Contract to check presence of fields in scraped items
@scrapes page_name page_body
"""
name = 'scrapes'
name = "scrapes"
def post_process(self, output):
for x in output:

173
scrapy/core/downloader/__init__.py
View File

@ -1,51 +1,61 @@
import random
from time import time
from datetime import datetime
from collections import deque
from datetime import datetime
from time import time
from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast
from twisted.internet import defer, task
from twisted.internet import task
from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.settings import BaseSettings
from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class Slot:
"""Downloader slot"""
def __init__(self, concurrency, delay, randomize_delay):
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
def __init__(self, concurrency: int, delay: float, randomize_delay: bool):
self.concurrency: int = concurrency
self.delay: float = delay
self.randomize_delay: bool = randomize_delay
self.active = set()
self.queue = deque()
self.transferring = set()
self.lastseen = 0
self.active: Set[Request] = set()
self.queue: Deque[Tuple[Request, Deferred]] = deque()
self.transferring: Set[Request] = set()
self.lastseen: float = 0
self.latercall = None
def free_transfer_slots(self):
def free_transfer_slots(self) -> int:
return self.concurrency - len(self.transferring)
def download_delay(self):
def download_delay(self) -> float:
if self.randomize_delay:
return random.uniform(0.5 * self.delay, 1.5 * self.delay)
return self.delay
def close(self):
def close(self) -> None:
if self.latercall and self.latercall.active():
self.latercall.cancel()
def __repr__(self):
def __repr__(self) -> str:
cls_name = self.__class__.__name__
return (f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})")
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})"
)
def __str__(self):
def __str__(self) -> str:
return (
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
@ -55,37 +65,45 @@ class Slot:
)
def _get_concurrency_delay(concurrency, spider, settings):
delay = settings.getfloat('DOWNLOAD_DELAY')
if hasattr(spider, 'download_delay'):
def _get_concurrency_delay(
concurrency: int, spider: Spider, settings: BaseSettings
) -> Tuple[int, float]:
delay: float = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
delay = spider.download_delay
if hasattr(spider, 'max_concurrent_requests'):
if hasattr(spider, "max_concurrent_requests"):
concurrency = spider.max_concurrent_requests
return concurrency, delay
class Downloader:
DOWNLOAD_SLOT = "download_slot"
DOWNLOAD_SLOT = 'download_slot'
def __init__(self, crawler):
self.settings = crawler.settings
self.signals = crawler.signals
self.slots = {}
self.active = set()
self.handlers = DownloadHandlers(crawler)
self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
self._slot_gc_loop = task.LoopingCall(self._slot_gc)
def __init__(self, crawler: "Crawler"):
self.settings: BaseSettings = crawler.settings
self.signals: SignalManager = crawler.signals
self.slots: Dict[str, Slot] = {}
self.active: Set[Request] = set()
self.handlers: DownloadHandlers = DownloadHandlers(crawler)
self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS")
self.domain_concurrency: int = self.settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
)
self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc)
self._slot_gc_loop.start(60)
self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict(
"DOWNLOAD_SLOTS", {}
)
def fetch(self, request, spider):
def _deactivate(response):
def fetch(self, request: Request, spider: Spider) -> Deferred:
def _deactivate(response: Response) -> Response:
self.active.remove(request)
return response
@ -93,47 +111,57 @@ class Downloader:
dfd = self.middleware.download(self._enqueue_request, request, spider)
return dfd.addBoth(_deactivate)
def needs_backout(self):
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
def _get_slot(self, request, spider):
def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
key = self._get_slot_key(request, spider)
if key not in self.slots:
conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
slot_settings = self.per_slot_settings.get(key, {})
conc = (
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
)
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
self.slots[key] = Slot(conc, delay, self.randomize_delay)
conc, delay = (
slot_settings.get("concurrency", conc),
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot
return key, self.slots[key]
def _get_slot_key(self, request, spider):
def _get_slot_key(self, request: Request, spider: Spider) -> str:
if self.DOWNLOAD_SLOT in request.meta:
return request.meta[self.DOWNLOAD_SLOT]
return cast(str, request.meta[self.DOWNLOAD_SLOT])
key = urlparse_cached(request).hostname or ''
key = urlparse_cached(request).hostname or ""
if self.ip_concurrency:
key = dnscache.get(key, key)
return key
def _enqueue_request(self, request, spider):
def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
key, slot = self._get_slot(request, spider)
request.meta[self.DOWNLOAD_SLOT] = key
def _deactivate(response):
def _deactivate(response: Response) -> Response:
slot.active.remove(request)
return response
slot.active.add(request)
self.signals.send_catch_log(signal=signals.request_reached_downloader,
request=request,
spider=spider)
deferred = defer.Deferred().addBoth(_deactivate)
self.signals.send_catch_log(
signal=signals.request_reached_downloader, request=request, spider=spider
)
deferred = Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred))
self._process_queue(spider, slot)
return deferred
def _process_queue(self, spider, slot):
def _process_queue(self, spider: Spider, slot: Slot) -> None:
from twisted.internet import reactor
if slot.latercall and slot.latercall.active():
return
@ -143,7 +171,9 @@ class Downloader:
if delay:
penalty = delay - now + slot.lastseen
if penalty > 0:
slot.latercall = reactor.callLater(penalty, self._process_queue, spider, slot)
slot.latercall = reactor.callLater(
penalty, self._process_queue, spider, slot
)
return
# Process enqueued requests if there are free slots to transfer for this slot
@ -157,7 +187,7 @@ class Downloader:
self._process_queue(spider, slot)
break
def _download(self, slot, request, spider):
def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred:
# The order is very important for the following deferreds. Do not change!
# 1. Create the download deferred
@ -165,12 +195,15 @@ class Downloader:
# 2. Notify response_downloaded listeners about the recent download
# before querying queue for next request
def _downloaded(response):
self.signals.send_catch_log(signal=signals.response_downloaded,
response=response,
request=request,
spider=spider)
def _downloaded(response: Response) -> Response:
self.signals.send_catch_log(
signal=signals.response_downloaded,
response=response,
request=request,
spider=spider,
)
return response
dfd.addCallback(_downloaded)
# 3. After response arrives, remove the request from transferring
@ -179,22 +212,22 @@ class Downloader:
# middleware itself)
slot.transferring.add(request)
def finish_transferring(_):
def finish_transferring(_: Any) -> Any:
slot.transferring.remove(request)
self._process_queue(spider, slot)
self.signals.send_catch_log(signal=signals.request_left_downloader,
request=request,
spider=spider)
self.signals.send_catch_log(
signal=signals.request_left_downloader, request=request, spider=spider
)
return _
return dfd.addBoth(finish_transferring)
def close(self):
def close(self) -> None:
self._slot_gc_loop.stop()
for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
def _slot_gc(self, age: float = 60) -> None:
mintime = time() - age
for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime:

95
scrapy/core/downloader/contextfactory.py
View File

@ -1,16 +1,30 @@
import warnings
from typing import TYPE_CHECKING, Any, List, Optional
from OpenSSL import SSL
from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.internet.ssl import (
AcceptableCiphers,
CertificateOptions,
optionsForClientTLS,
platformTrust,
)
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
from scrapy.core.downloader.tls import (
DEFAULT_CIPHERS,
ScrapyClientTLSOptions,
openssl_methods,
)
from scrapy.settings import BaseSettings
from scrapy.utils.misc import create_instance, load_object
if TYPE_CHECKING:
from twisted.internet._sslverify import ClientTLSOptions
@implementer(IPolicyForHTTPS)
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
@ -21,25 +35,47 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
which allows TLS protocol negotiation
'A TLS/SSL connection established with [this method] may
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
def __init__(
self,
method: int = SSL.SSLv23_METHOD,
tls_verbose_logging: bool = False,
tls_ciphers: Optional[str] = None,
*args: Any,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
self._ssl_method: int = method
self.tls_verbose_logging: bool = tls_verbose_logging
self.tls_ciphers: AcceptableCiphers
if tls_ciphers:
self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
else:
self.tls_ciphers = DEFAULT_CIPHERS
@classmethod
def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
def from_settings(
cls,
settings: BaseSettings,
method: int = SSL.SSLv23_METHOD,
*args: Any,
**kwargs: Any,
):
tls_verbose_logging: bool = settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
return cls( # type: ignore[misc]
method=method,
tls_verbose_logging=tls_verbose_logging,
tls_ciphers=tls_ciphers,
*args,
**kwargs,
)
def getCertificateOptions(self):
def getCertificateOptions(self) -> CertificateOptions:
# setting verify=True will require you to provide CAs
# to verify against; in other words: it's not that simple
@ -53,19 +89,24 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
method=getattr(self, "method", getattr(self, "_ssl_method", None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
)
# kept for old-style HTTP/1.0 downloader context twisted calls,
# e.g. connectSSL()
def getContext(self, hostname=None, port=None):
return self.getCertificateOptions().getContext()
def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context:
ctx = self.getCertificateOptions().getContext()
ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT
return ctx
def creatorForNetloc(self, hostname, port):
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
verbose_logging=self.tls_verbose_logging)
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
return ScrapyClientTLSOptions(
hostname.decode("ascii"),
self.getContext(),
verbose_logging=self.tls_verbose_logging,
)
@implementer(IPolicyForHTTPS)
@ -87,7 +128,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
def creatorForNetloc(self, hostname, port):
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@ -95,7 +136,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
return optionsForClientTLS(
hostname=hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
extraCertificateOptions={"method": self._ssl_method},
)
@ -106,20 +147,22 @@ class AcceptableProtocolsContextFactory:
negotiation.
"""
def __init__(self, context_factory, acceptable_protocols):
def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]):
verifyObject(IPolicyForHTTPS, context_factory)
self._wrapped_context_factory = context_factory
self._acceptable_protocols = acceptable_protocols
self._wrapped_context_factory: Any = context_factory
self._acceptable_protocols: List[bytes] = acceptable_protocols
def creatorForNetloc(self, hostname, port):
options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(
hostname, port
)
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)
return options
def load_context_factory_from_settings(settings, crawler):
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
# try method-aware context factory
try:
context_factory = create_instance(

53
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,35 +1,41 @@
"""Download handlers for different schemes"""
import logging
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast
from twisted.internet import defer
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class DownloadHandlers:
def __init__(self, crawler):
self._crawler = crawler
self._schemes = {} # stores acceptable schemes on instancing
self._handlers = {} # stores instanced handlers for schemes
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
def __init__(self, crawler: "Crawler"):
self._crawler: "Crawler" = crawler
self._schemes: Dict[
str, Union[str, Callable]
] = {} # stores acceptable schemes on instancing
self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes
self._notconfigured: Dict[str, str] = {} # remembers failed handlers
handlers: Dict[str, Union[str, Callable]] = without_none_values(
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
)
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
crawler.signals.connect(self._close, signals.engine_stopped)
def _get_handler(self, scheme):
def _get_handler(self, scheme: str) -> Any:
"""Lazy-load the downloadhandler for a scheme
only on the first request for that scheme.
"""
@ -38,16 +44,16 @@ class DownloadHandlers:
if scheme in self._notconfigured:
return None
if scheme not in self._schemes:
self._notconfigured[scheme] = 'no handler available for that scheme'
self._notconfigured[scheme] = "no handler available for that scheme"
return None
return self._load_handler(scheme)
def _load_handler(self, scheme, skip_lazy=False):
def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any:
path = self._schemes[scheme]
try:
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, 'lazy', True):
if skip_lazy and getattr(dhcls, "lazy", True):
return None
dh = create_instance(
objcls=dhcls,
@ -58,24 +64,29 @@ class DownloadHandlers:
self._notconfigured[scheme] = str(ex)
return None
except Exception as ex:
logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True, extra={'crawler': self._crawler})
logger.error(
'Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
exc_info=True,
extra={"crawler": self._crawler},
)
self._notconfigured[scheme] = str(ex)
return None
else:
self._handlers[scheme] = dh
return dh
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
return handler.download_request(request, spider)
raise NotSupported(
f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
)
return cast(Deferred, handler.download_request(request, spider))
@defer.inlineCallbacks
def _close(self, *_a, **_kw):
def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]:
for dh in self._handlers.values():
if hasattr(dh, 'close'):
if hasattr(dh, "close"):
yield dh.close()

7
scrapy/core/downloader/handlers/datauri.py
View File

@ -14,9 +14,8 @@ class DataURIDownloadHandler:
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse)
and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
charset = uri.media_type_parameters.get("charset")
resp_kwargs["encoding"] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)

5
scrapy/core/downloader/handlers/file.py
View File

@ -1,3 +1,5 @@
from pathlib import Path
from w3lib.url import file_uri_to_path
from scrapy.responsetypes import responsetypes
@ -10,7 +12,6 @@ class FileDownloadHandler:
@defers
def download_request(self, request, spider):
filepath = file_uri_to_path(request.url)
with open(filepath, 'rb') as fo:
body = fo.read()
body = Path(filepath).read_bytes()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

Some files were not shown because too many files have changed in this diff Show More