RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 3689-update

This commit is contained in:
Andrey Rahmatullin 2022-11-07 16:33:21 +05:00 committed by GitHub
parent 0c24cdb257 77cd511a5e
commit b03d84e9fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
313 changed files with 17033 additions and 4451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.bandit.yml
View File

@ -8,6 +8,7 @@ skips:
- B311
- B320
- B321
- B324
- B402 # https://github.com/scrapy/scrapy/issues/4180
- B403
- B404

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.3.0
current_version = 2.7.1
commit = True
tag = True
tag_name = {new_version}

22
.flake8 Normal file
View File

@ -0,0 +1,22 @@
[flake8]
max-line-length = 119
ignore = W503
exclude =
docs/conf.py
per-file-ignores =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py:E402
scrapy/core/downloader/handlers/http.py:F401
scrapy/http/__init__.py:F401
scrapy/linkextractors/__init__.py:E402,F401
scrapy/selector/__init__.py:F401
scrapy/spiders/__init__.py:E402,F401
# Issues pending a review:
scrapy/utils/url.py:F403,F405
tests/test_loader.py:E741

44
.github/workflows/checks.yml vendored Normal file
View File

@ -0,0 +1,44 @@
name: Checks
on: [push, pull_request]
jobs:
checks:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: "3.11"
env:
TOXENV: security
- python-version: "3.11"
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: pylint
- python-version: 3.7
env:
TOXENV: typing
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
- python-version: "3.11"
env:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Run check
env: ${{ matrix.env }}
run: |
pip install -U tox
tox

31
.github/workflows/publish.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: Publish
on: [push]
jobs:
publish:
runs-on: ubuntu-latest
if: startsWith(github.event.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Check Tag
id: check-release-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
echo ::set-output name=release_tag::true
fi
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*

26
.github/workflows/tests-macos.yml vendored Normal file
View File

@ -0,0 +1,26 @@
name: macOS
on: [push, pull_request]
jobs:
tests:
runs-on: macos-11
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Run tests
run: |
pip install -U tox
tox -e py
- name: Upload coverage report
run: bash <(curl -s https://codecov.io/bash)

69
.github/workflows/tests-ubuntu.yml vendored Normal file
View File

@ -0,0 +1,69 @@
name: Ubuntu
on: [push, pull_request]
jobs:
tests:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.11"
env:
TOXENV: asyncio
- python-version: pypy3.9
env:
TOXENV: pypy3
# pinned deps
- python-version: 3.7.13
env:
TOXENV: pinned
- python-version: 3.7.13
env:
TOXENV: asyncio-pinned
- python-version: pypy3.7
env:
TOXENV: pypy3-pinned
# extras
# extra-deps includes reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: extra-deps
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
- name: Upload coverage report
run: bash <(curl -s https://codecov.io/bash)

46
.github/workflows/tests-windows.yml vendored Normal file
View File

@ -0,0 +1,46 @@
name: Windows
on: [push, pull_request]
jobs:
tests:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: 3.7
env:
TOXENV: windows-pinned
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: asyncio
# no binary package for lxml for 3.11 yet
# - python-version: "3.11"
# env:
# TOXENV: py
# - python-version: "3.11"
# env:
# TOXENV: asyncio
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox

5
.gitignore vendored
View File

@ -14,6 +14,8 @@ htmlcov/
.coverage
.pytest_cache/
.coverage.*
coverage.*
test-output.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt
@ -21,3 +23,6 @@ htmlcov/
# Windows
Thumbs.db
# OSX miscellaneous
.DS_Store

11
.readthedocs.yml
View File

@ -3,10 +3,15 @@ formats: all
sphinx:
configuration: docs/conf.py
fail_on_warning: true
build:
os: ubuntu-20.04
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: "3.11" # Keep in sync with .github/workflows/checks.yml
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.7 # Keep in sync with .travis.yml
install:
- requirements: docs/requirements.txt
- path: .

75
.travis.yml
View File

@ -1,75 +0,0 @@
language: python
dist: xenial
branches:
only:
- master
- /^\d\.\d+$/
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
- env: TOXENV=security
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pylint
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=typing
python: 3.8
- env: TOXENV=pinned
python: 3.6.1
- env: TOXENV=asyncio-pinned
python: 3.6.1
- env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
- env: TOXENV=py
python: 3.6
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
- env: TOXENV=py
python: 3.7
- env: TOXENV=py PYPI_RELEASE_JOB=true
python: 3.8
dist: bionic
- env: TOXENV=extra-deps
python: 3.8
dist: bionic
- env: TOXENV=asyncio
python: 3.8
dist: bionic
install:
- |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
- pip install -U tox twine wheel codecov
script: tox
after_success:
- codecov
notifications:
irc:
use_notice: true
skip_join: true
channels:
- irc.freenode.org#scrapy
cache:
directories:
- $HOME/.cache/pip
deploy:
provider: pypi
distributions: "sdist bdist_wheel"
user: scrapy
password:
secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
on:
tags: true
repo: scrapy/scrapy
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

4
AUTHORS
View File

@ -1,8 +1,8 @@
Scrapy was brought to life by Shane Evans while hacking a scraping framework
prototype for Mydeco (mydeco.com). It soon became maintained, extended and
improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to
bootstrap the project. In mid-2011, Scrapinghub became the new official
maintainer.
bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new
official maintainer.
Here is the list of the primary authors & contributors:

151
CODE_OF_CONDUCT.md
View File

@ -1,74 +1,133 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
Examples of behavior that contributes to a positive environment for our
community include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior by participants include:
Examples of unacceptable behavior include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
## Enforcement Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at opensource@scrapinghub.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
reported to the community leaders responsible for enforcement at
opensource@zyte.com.
All complaints will be reviewed and investigated promptly and fairly.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version].
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations

4
INSTALL
View File

@ -1,4 +0,0 @@
For information about installing Scrapy see:
* docs/intro/install.rst (local file)
* https://docs.scrapy.org/en/latest/intro/install.html (online version)

4
INSTALL.md Normal file
View File

@ -0,0 +1,4 @@
For information about installing Scrapy see:
* [Local docs](docs/intro/install.rst)
* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)

35
README.rst
View File

@ -1,3 +1,6 @@
.. image:: https://scrapy.org/img/scrapylogo.png
:target: https://scrapy.org/
======
Scrapy
======
@ -10,9 +13,17 @@ Scrapy
:target: https://pypi.python.org/pypi/Scrapy
:alt: Supported Python Versions
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
:target: https://travis-ci.org/scrapy/scrapy
:alt: Build Status
.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
:alt: Windows
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
:target: https://pypi.python.org/pypi/Scrapy
@ -34,19 +45,28 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other
contributors`_.
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
.. _Zyte: https://www.zyte.com/
Check the Scrapy homepage at https://scrapy.org for more information,
including a list of features.
Requirements
============
* Python 3.6+
* Python 3.7+
* Works on Linux, Windows, macOS, BSD
Install
=======
The quick way::
The quick way:
.. code:: bash
pip install scrapy
@ -77,11 +97,10 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
Code of Conduct
---------------
Please note that this project is released with a Contributor Code of Conduct
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.
By participating in this project you agree to abide by its terms.
Please report unacceptable behavior to opensource@scrapinghub.com.
Please report unacceptable behavior to opensource@zyte.com.
Companies using Scrapy
======================

22
azure-pipelines.yml
View File

@ -1,22 +0,0 @@
variables:
TOXENV: py
pool:
vmImage: 'windows-latest'
strategy:
matrix:
Python36:
python.version: '3.6'
TOXENV: windows-pinned
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- script: |
pip install -U tox twine wheel codecov
tox
displayName: 'Run test suite'

45
conftest.py
View File

@ -1,6 +1,9 @@
from pathlib import Path
import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys
@ -18,10 +21,19 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
with open('tests/ignores.txt') as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
*_py_files("scrapy/core/http2"),
)
)
@pytest.fixture()
@ -30,14 +42,12 @@ def chdir(tmpdir):
tmpdir.chdir()
def pytest_collection_modifyitems(session, config, items):
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
try:
from pytest_flake8 import Flake8Item
if config.getoption('--flake8'):
items[:] = [item for item in items if isinstance(item, Flake8Item)]
except ImportError:
pass
def pytest_addoption(parser):
parser.addoption(
"--reactor",
default="default",
choices=["default", "asyncio"],
)
@pytest.fixture(scope='class')
@ -55,5 +65,16 @@ def only_asyncio(request, reactor_pytest):
pytest.skip('This test is only run with --reactor=asyncio')
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
pytest.skip('This test is only run without --reactor=asyncio')
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
# Generate localhost certificate files, needed by some tests
generate_keys()

2
docs/Makefile
View File

@ -8,7 +8,7 @@ PYTHON = python
SPHINXOPTS =
PAPER =
SOURCES =
SHELL = /bin/bash
SHELL = /usr/bin/env bash
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
-D latex_elements.papersize=$(PAPER) \

2
docs/README.rst
View File

@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
Start over
----------
To cleanup all generated documentation files and start from scratch run::
To clean up all generated documentation files and start from scratch run::
make clean

26
docs/_ext/scrapydocs.py
View File

@ -15,7 +15,7 @@ class SettingsListDirective(Directive):
def is_setting_index(node):
if node.tagname == 'index':
if node.tagname == 'index' and node['entries']:
# index entries for setting directives look like:
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
def setup(app):
app.add_crossref_type(
directivename = "setting",
rolename = "setting",
indextemplate = "pair: %s; setting",
directivename="setting",
rolename="setting",
indextemplate="pair: %s; setting",
)
app.add_crossref_type(
directivename = "signal",
rolename = "signal",
indextemplate = "pair: %s; signal",
directivename="signal",
rolename="signal",
indextemplate="pair: %s; signal",
)
app.add_crossref_type(
directivename = "command",
rolename = "command",
indextemplate = "pair: %s; command",
directivename="command",
rolename="command",
indextemplate="pair: %s; command",
)
app.add_crossref_type(
directivename = "reqmeta",
rolename = "reqmeta",
indextemplate = "pair: %s; reqmeta",
directivename="reqmeta",
rolename="reqmeta",
indextemplate="pair: %s; reqmeta",
)
app.add_role('source', source_role)
app.add_role('commit', commit_role)

10
docs/_static/custom.css vendored Normal file
View File

@ -0,0 +1,10 @@
/* Move lists closer to their introducing paragraph */
.rst-content .section ol p, .rst-content .section ul p {
margin-bottom: 0px;
}
.rst-content p + ol, .rst-content p + ul {
margin-top: -18px; /* Compensates margin-top: 24px of p */
}
.rst-content dl p + ol, .rst-content dl p + ul {
margin-top: -6px; /* Compensates margin-top: 12px of p */
}

31
docs/_static/selectors-sample1.html vendored
View File

@ -1,16 +1,17 @@
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
</div>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
</div>
</body>
</html>

7
docs/_templates/layout.html vendored
View File

@ -3,14 +3,9 @@
{% block footer %}
{{ super() }}
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();
analytics.ready(function () {
ga('require', 'linker');
ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
ga('linker:autoLink', ['zyte.com']);
});
</script>
{% endblock %}

19
docs/conf.py
View File

@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme'
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
@ -183,6 +182,10 @@ html_copy_source = True
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
html_css_files = [
'custom.css',
]
# Options for LaTeX output
# ------------------------
@ -269,7 +272,6 @@ coverage_ignore_pyobjects = [
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.item\.DictItem$',
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
# Implementation detail of LxmlLinkExtractor
@ -283,15 +285,18 @@ coverage_ignore_pyobjects = [
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cryptography' : ('https://cryptography.io/en/latest/', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
'tox': ('https://tox.wiki/en/latest/', None),
'twisted': ('https://docs.twisted.org/en/stable/', None),
'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
}
intersphinx_disabled_reftypes = []
# Options for sphinx-hoverxref options
@ -300,10 +305,14 @@ intersphinx_mapping = {
hoverxref_auto_ref = True
hoverxref_role_types = {
"class": "tooltip",
"command": "tooltip",
"confval": "tooltip",
"hoverxref": "tooltip",
"mod": "tooltip",
"ref": "tooltip",
"reqmeta": "tooltip",
"setting": "tooltip",
"signal": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']

8
docs/conftest.py
View File

@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.codeblock import CodeBlockParser
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
@ -21,7 +25,7 @@ def setup(namespace):
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
CodeBlockParser(future_imports=['print_function']),
PythonCodeBlockParser(future_imports=['print_function']),
skip,
],
pattern='*.rst',

14
docs/contributing.rst
View File

@ -140,7 +140,7 @@ original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
not considered rude as long as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch
@ -214,7 +214,7 @@ Tests
=====
Tests are implemented using the :doc:`Twisted unit-testing framework
<twisted:core/development/policy/test-standard>`. Running tests requires
<twisted:development/test-standard>`. Running tests requires
:doc:`tox <tox:index>`.
.. _running-tests:
@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
the tests with Python 3.6 use::
the tests with Python 3.7 use::
tox -e py36
tox -e py37
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py36,py38 -p auto
tox -e py37,py38 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
tox -e py36 -- scrapy tests -n auto
tox -e py37 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:

69
docs/faq.rst
View File

@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
See :ref:`topics-request-response-ref-request-callback-arguments`.
Scrapy crashes with: ImportError: No module named win32api
----------------------------------------------------------
You need to install `pywin32`_ because of `this Twisted bug`_.
.. _pywin32: https://sourceforge.net/projects/pywin32/
.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
How can I simulate a user login in my spider?
---------------------------------------------
@ -145,6 +136,41 @@ How can I make Scrapy consume less memory?
See previous question.
How can I prevent memory errors due to many allowed domains?
------------------------------------------------------------
If you have a spider with a long list of
:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider
replacing the default
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
less memory. For example:
- If your domain names are similar enough, use your own regular expression
instead joining the strings in
:attr:`~scrapy.Spider.allowed_domains` into a complex regular
expression.
- If you can `meet the installation requirements`_, use pyre2_ instead of
Python’s re_ to compile your URL-filtering regular expression. See
:issue:`1908`.
See also other suggestions at `StackOverflow`_.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation::
SPIDER_MIDDLEWARES = {
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
'myproject.middlewares.CustomOffsiteMiddleware': 500,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
.. _pyre2: https://github.com/andreasvc/pyre2
.. _re: https://docs.python.org/library/re.html
.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
Can I use Basic HTTP Authentication in my spiders?
--------------------------------------------------
@ -315,6 +341,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items
You may need to remove namespaces. See :ref:`removing-namespaces`.
.. _faq-split-item:
How to split an item into multiple items in an item pipeline?
@ -363,15 +390,27 @@ How can I cancel the download of a given response?
--------------------------------------------------
In some situations, it might be useful to stop the download of a certain response.
For instance, if you only need the first part of a large response and you would like
to save resources by avoiding the download of the whole body.
In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
the :ref:`topics-stop-response-download` topic for additional information and examples.
For instance, sometimes you can determine whether or not you need the full contents
of a response by inspecting its headers or the first bytes of its body. In that case,
you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received`
or :class:`~scrapy.signals.headers_received` signals and raising a
:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the
:ref:`topics-stop-response-download` topic for additional information and examples.
Running ``runspider`` I get ``error: No spider found in file: <filename>``
--------------------------------------------------------------------------
This may happen if your Scrapy project has a spider module with a name that
conflicts with the name of one of the `Python standard library modules`_, such
as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
See :issue:`2680`.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _Python standard library modules: https://docs.python.org/py-modindex.html
.. _Python package: https://pypi.org/
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search

26
docs/index.rst
View File

@ -12,6 +12,8 @@ testing.
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
.. _getting-help:
Getting help
============
@ -24,12 +26,14 @@ Having trouble? We'd like to help!
* Search for questions on the archives of the `scrapy-users mailing list`_.
* Ask a question in the `#scrapy IRC channel`_,
* Report bugs with Scrapy in our `issue tracker`_.
* Join the Discord community `Scrapy Discord`_.
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
First steps
@ -78,7 +82,6 @@ Basic concepts
topics/settings
topics/exceptions
:doc:`topics/commands`
Learn about the command-line tool used to manage your Scrapy project.
@ -127,7 +130,6 @@ Built-in services
topics/stats
topics/email
topics/telnetconsole
topics/webservice
:doc:`topics/logging`
Learn how to use Python's builtin logging on Scrapy.
@ -141,9 +143,6 @@ Built-in services
:doc:`topics/telnetconsole`
Inspect a running crawler using a built-in Python console.
:doc:`topics/webservice`
Monitor and control a crawler using a web service.
Solving specific problems
=========================
@ -226,9 +225,11 @@ Extending Scrapy
topics/downloader-middleware
topics/spider-middleware
topics/extensions
topics/api
topics/signals
topics/scheduler
topics/exporters
topics/components
topics/api
:doc:`topics/architecture`
@ -243,15 +244,22 @@ Extending Scrapy
:doc:`topics/extensions`
Extend Scrapy with your custom functionality
:doc:`topics/api`
Use it on extensions and middlewares to extend Scrapy functionality
:doc:`topics/signals`
See all available signals and how to work with them.
:doc:`topics/scheduler`
Understand the scheduler component.
:doc:`topics/exporters`
Quickly export your scraped items to a file (XML, CSV, etc).
:doc:`topics/components`
Learn the common API and some good practices when building custom Scrapy
components.
:doc:`topics/api`
Use it on extensions and middlewares to extend Scrapy functionality.
All the rest
============

2
docs/intro/examples.rst
View File

@ -7,7 +7,7 @@ Examples
The best way to learn is with examples, and Scrapy is no exception. For this
reason, there is an example Scrapy project named quotesbot_, that you can use to
play and learn more about Scrapy. It contains two spiders for
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
expressions.
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.

65
docs/intro/install.rst
View File

@ -9,9 +9,10 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
Scrapy requires Python 3.7+, either the CPython implementation (default) or
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
Installing Scrapy
=================
@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with::
pip install Scrapy
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
For more detailed and platform specifics instructions, as well as
troubleshooting information, read on.
@ -51,17 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
* `twisted`_, an asynchronous networking framework
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
The minimal versions which Scrapy is tested against are:
* Twisted 14.0
* lxml 3.4
* pyOpenSSL 0.14
Scrapy may work with older versions of these packages
but it is not guaranteed it will continue working
because it’s not being tested against them.
Some of these packages themselves depends on non-Python packages
Some of these packages themselves depend on non-Python packages
that might require additional installation steps depending on your platform.
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
@ -69,10 +60,9 @@ In case of any trouble related to these dependencies,
please refer to their respective installation instructions:
* `lxml installation`_
* `cryptography installation`_
* :doc:`cryptography installation <cryptography:installation>`
.. _lxml installation: https://lxml.de/installation.html
.. _cryptography installation: https://cryptography.io/en/latest/installation/
.. _intro-using-virtualenv:
@ -118,6 +108,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
conda install -c conda-forge scrapy
To install Scrapy on Windows using ``pip``:
.. warning::
This installation method requires “Microsoft Visual C++” for installing some
Scrapy dependencies, which demands significantly more disk space than Anaconda.
#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer.
#. Run the Visual Studio Installer.
#. Under the Workloads section, select **C++ build tools**.
#. Check the installation details and make sure following packages are selected as optional components:
* **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) )
* **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0))
#. Install the Visual Studio Build Tools.
Now, you should be able to :ref:`install Scrapy <intro-install-scrapy>` using ``pip``.
.. _intro-install-ubuntu:
@ -169,14 +180,14 @@ prevents ``pip`` from updating system packages. This has to be addressed to
successfully install Scrapy and its dependencies. Here are some proposed
solutions:
* *(Recommended)* **Don't** use system python, install a new, updated version
* *(Recommended)* **Don't** use system Python. Install a new, updated version
that doesn't conflict with the rest of your system. Here's how to do it using
the `homebrew`_ package manager:
* Install `homebrew`_ following the instructions in https://brew.sh/
* Update your ``PATH`` variable to state that homebrew packages should be
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
if you're using `zsh`_ as default shell)::
echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
@ -208,13 +219,13 @@ After any of these workarounds you should be able to install Scrapy::
PyPy
----
We recommend using the latest PyPy version. The version tested is 5.9.0.
We recommend using the latest PyPy version.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
This means that these dependencies will be built during installation.
On macOS, you are likely to face an issue with building the Cryptography
dependency. The solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
recommends (only needed when installing Scrapy). Installing on Linux has no special
@ -265,10 +276,10 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
.. _cryptography: https://cryptography.io/en/latest/
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
.. _setuptools: https://pypi.python.org/pypi/setuptools
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
.. _homebrew: https://brew.sh/
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
.. _conda-forge: https://conda-forge.org/

10
docs/intro/overview.rst
View File

@ -4,7 +4,7 @@
Scrapy at a glance
==================
Scrapy is an application framework for crawling web sites and extracting
Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
structured data which can be used for a wide range of useful applications, like
data mining, information processing or historical archival.
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
example of a Scrapy Spider using the simplest way to run a spider.
Here's the code for a spider that scrapes famous quotes from website
http://quotes.toscrape.com, following the pagination::
https://quotes.toscrape.com, following the pagination::
import scrapy
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
class QuotesSpider(scrapy.Spider):
name = 'quotes'
start_urls = [
'http://quotes.toscrape.com/tag/humor/',
'https://quotes.toscrape.com/tag/humor/',
]
def parse(self, response):
@ -45,9 +45,9 @@ http://quotes.toscrape.com, following the pagination::
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.jl
scrapy runspider quotes_spider.py -o quotes.jsonl
When this finishes you will have in the ``quotes.jl`` file a list of the
When this finishes you will have in the ``quotes.jsonl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}

106
docs/intro/tutorial.rst
View File

@ -7,7 +7,7 @@ Scrapy Tutorial
In this tutorial, we'll assume that Scrapy is already installed on your system.
If that's not the case, see :ref:`intro-install`.
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
that lists quotes from famous authors.
This tutorial will walk you through these tasks:
@ -78,7 +78,7 @@ Our first Spider
Spiders are classes that you define and that Scrapy uses to scrape information
from a website (or a group of websites). They must subclass
:class:`~scrapy.spiders.Spider` and define the initial requests to make,
:class:`~scrapy.Spider` and define the initial requests to make,
optionally how to follow links in the pages, and how to parse the downloaded
page content to extract data.
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
def start_requests(self):
urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named
self.log(f'Saved file {filename}')
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
and defines some attributes and methods:
* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be
unique within a project, that is, you can't set the same name for different
Spiders.
* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of
* :meth:`~scrapy.Spider.start_requests`: must return an iterable of
Requests (you can return a list of requests or write a generator function)
which the Spider will begin to crawl from. Subsequent requests will be
generated successively from these initial requests.
* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
* :meth:`~scrapy.Spider.parse`: a method that will be called to handle
the response downloaded for each of the requests made. The response parameter
is an instance of :class:`~scrapy.http.TextResponse` that holds
the page content and has further helpful methods to handle it.
The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting
the scraped data as dicts and also finding new URLs to
follow and creating new requests (:class:`~scrapy.http.Request`) from them.
follow and creating new requests (:class:`~scrapy.Request`) from them.
How to run our spider
---------------------
@ -143,9 +143,9 @@ similar to this::
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs.
What just happened under the hood?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects
returned by the ``start_requests`` method of the Spider. Upon receiving a
response for each one, it instantiates :class:`~scrapy.http.Response` objects
and calls the callback method associated with the request (in this case, the
@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the
A shortcut to the start_requests method
---------------------------------------
Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method
that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs,
you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute
Instead of implementing a :meth:`~scrapy.Spider.start_requests` method
that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
import scrapy
@ -184,8 +184,8 @@ for your spider::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
]
def parse(self, response):
@ -194,9 +194,9 @@ for your spider::
with open(filename, 'wb') as f:
f.write(response.body)
The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy
to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's
to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's
default callback method, which is called for requests without an explicitly
assigned callback.
@ -207,7 +207,7 @@ Extracting data
The best way to learn how to extract data with Scrapy is trying selectors
using the :ref:`Scrapy shell <topics-shell>`. Run::
scrapy shell 'http://quotes.toscrape.com/page/1/'
scrapy shell 'https://quotes.toscrape.com/page/1/'
.. note::
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
On Windows, use double quotes instead::
scrapy shell "http://quotes.toscrape.com/page/1/"
scrapy shell "https://quotes.toscrape.com/page/1/"
You will see something like::
[ ... Scrapy log here ... ]
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
[s] Available Scrapy objects:
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
[s] item {}
[s] request <GET http://quotes.toscrape.com/page/1/>
[s] response <200 http://quotes.toscrape.com/page/1/>
[s] request <GET https://quotes.toscrape.com/page/1/>
[s] response <200 https://quotes.toscrape.com/page/1/>
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
[s] Useful shortcuts:
@ -241,14 +241,14 @@ object:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements
:class:`~scrapy.Selector` objects that wrap around XML/HTML elements
and allow you to run further queries to fine-grain the selection or extract the
data.
@ -277,9 +277,19 @@ As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
find any element matching the selection.
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
raise an :exc:`IndexError` exception if there are no results::
>>> response.css('noelement')[0].get()
Traceback (most recent call last):
...
IndexError: list index out of range
You might want to use ``.get()`` directly on the
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
if there are no results::
>>> response.css("noelement").get()
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
@ -345,7 +355,7 @@ Extracting quotes and authors
Now that you know a bit about selection and extraction, let's complete our
spider by writing the code to extract the quotes from the web page.
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
like this:
.. code-block:: html
@ -369,7 +379,7 @@ like this:
Let's open up scrapy shell and play a bit to find out how to extract the data
we want::
$ scrapy shell 'http://quotes.toscrape.com'
scrapy shell 'https://quotes.toscrape.com'
We get a list of selectors for the quote HTML elements with:
@ -434,8 +444,8 @@ in the callback, as you can see below::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
]
def parse(self, response):
@ -448,9 +458,9 @@ in the callback, as you can see below::
If you run this spider, it will output the extracted data with the log::
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
@ -464,7 +474,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports
scrapy crawl quotes -O quotes.json
That will generate an ``quotes.json`` file containing all scraped items,
That will generate a ``quotes.json`` file containing all scraped items,
serialized in `JSON`_.
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
@ -472,13 +482,13 @@ to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
scrapy crawl quotes -o quotes.jsonl
The `JSON Lines`_ format is useful because it's stream-like, you can easily
append new records to it. It doesn't have the same problem of JSON when you run
twice. Also, as each record is a separate line, you can process big files
without having to fit everything in memory, there are tools like `JQ`_ to help
doing that at the command-line.
do that at the command-line.
In small projects (like the one in this tutorial), that should be enough.
However, if you want to perform more complex things with the scraped items, you
@ -495,7 +505,7 @@ Following links
===============
Let's say, instead of just scraping the stuff from the first two pages
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
Now that you know how to extract data from pages, let's see how to follow links
from them.
@ -539,7 +549,7 @@ page, extracting data from it::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/1/',
]
def parse(self, response):
@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/1/',
]
def parse(self, response):
@ -644,7 +654,7 @@ this time for scraping author information::
class AuthorSpider(scrapy.Spider):
name = 'author'
start_urls = ['http://quotes.toscrape.com/']
start_urls = ['https://quotes.toscrape.com/']
def parse(self, response):
author_page_links = response.css('.author + a')
@ -670,7 +680,7 @@ the pagination links with the ``parse`` callback as we saw before.
Here we're passing callbacks to
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
arguments to make the code shorter; it also works for
:class:`~scrapy.http.Request`.
:class:`~scrapy.Request`.
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.
@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument::
name = "quotes"
def start_requests(self):
url = 'http://quotes.toscrape.com/'
url = 'https://quotes.toscrape.com/'
tag = getattr(self, 'tag', None)
if tag is not None:
url = url + 'tag/' + tag
@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument::
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
will only visit URLs from the ``humor`` tag, such as
``http://quotes.toscrape.com/tag/humor``.
``https://quotes.toscrape.com/tag/humor``.
You can :ref:`learn more about handling spider arguments here <spiderargs>`.

1468
docs/news.rst
View File

File diff suppressed because it is too large Load Diff

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx_rtd_theme>=0.4
sphinx==5.0.2
sphinx-hoverxref==1.1.1
sphinx-notfound-page==0.8
sphinx-rtd-theme==1.0.0

11
docs/topics/api.rst
View File

@ -29,9 +29,16 @@ how you :ref:`configure the downloader middlewares
.. class:: Crawler(spidercls, settings)
The Crawler object must be instantiated with a
:class:`scrapy.spiders.Spider` subclass and a
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
.. attribute:: request_fingerprinter
The request fingerprint builder of this crawler.
This is used from extensions and middlewares to build short, unique
identifiers for requests. See :ref:`request-fingerprints`.
.. attribute:: settings
The settings manager of this crawler.
@ -196,7 +203,7 @@ SpiderLoader API
match the request's url against the domains of the spiders.
:param request: queried request
:type request: :class:`~scrapy.http.Request` instance
:type request: :class:`~scrapy.Request` instance
.. _topics-api-signals:

7
docs/topics/architecture.rst
View File

@ -67,7 +67,7 @@ this:
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
to crawl.
9. The process repeats (from step 1) until there are no more requests from the
9. The process repeats (from step 3) until there are no more requests from the
:ref:`Scheduler <component-scheduler>`.
Components
@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the
Scheduler
---------
The Scheduler receives requests from the engine and enqueues them for feeding
them later (also to the engine) when the engine requests them.
The :ref:`scheduler <topics-scheduler>` receives requests from the engine and
enqueues them for feeding them later (also to the engine) when the engine
requests them.
.. _component-downloader:

92
docs/topics/asyncio.rst
View File

@ -1,16 +1,15 @@
.. _using-asyncio:
=======
asyncio
=======
.. versionadded:: 2.0
Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
reactor <install-asyncio>`, you may use :mod:`asyncio` and
Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related changes without a deprecation
period or warning.
.. _install-asyncio:
@ -27,6 +26,7 @@ reactor manually. You can do that using
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
Using custom asyncio loops
@ -37,4 +37,86 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
use it instead of the default asyncio event loop.
.. _asyncio-windows:
Windows-specific notes
======================
The Windows implementation of :mod:`asyncio` can use two event loop
implementations:
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
when using Twisted.
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
with Twisted.
So on Python 3.8+ the event loop class needs to be changed.
.. versionchanged:: 2.6.0
The event loop class is changed automatically when you change the
:setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`.
To change the event loop class manually, call the following code before
installing the reactor::
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
You can put this in the same function that installs the reactor, if you do that
yourself, or in some code that runs before the reactor is installed, e.g.
``settings.py``.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
.. _asyncio-await-dfd:
Awaiting on Deferreds
=====================
When the asyncio reactor isn't installed, you can await on Deferreds in the
coroutines directly. When it is installed, this is not possible anymore, due to
specifics of the Scrapy coroutine integration (the coroutines are wrapped into
:class:`asyncio.Future` objects, not into
:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
Futures. Scrapy provides two helpers for this:
.. autofunction:: scrapy.utils.defer.deferred_to_future
.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
.. tip:: If you need to use these functions in code that aims to be compatible
with lower versions of Scrapy that do not provide these functions,
down to Scrapy 2.0 (earlier versions do not support
:mod:`asyncio`), you can copy the implementation of these functions
into your own code.
.. _enforce-asyncio-requirement:
Enforcing asyncio as a requirement
==================================
If you are writing a :ref:`component <topics-components>` that requires asyncio
to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
:ref:`enforce it as a requirement <enforce-component-requirements>`. For
example::
from scrapy.utils.reactor import is_asyncio_reactor_installed
class MyComponent:
def __init__(self):
if not is_asyncio_reactor_installed():
raise ValueError(
f"{MyComponent.__qualname__} requires the asyncio Twisted "
f"reactor. Make sure you have it configured in the "
f"TWISTED_REACTOR setting. See the asyncio documentation "
f"of Scrapy for more information."
)

5
docs/topics/benchmarking.rst
View File

@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which
results in slower crawl rates. How slower depends on how much your spider does
and how well it's written.
In the future, more cases will be added to the benchmarking suite to cover
other common scenarios.
Use scrapy-bench_ for more complex benchmarking.
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench

32
docs/topics/commands.rst
View File

@ -230,10 +230,16 @@ Usage example::
genspider
---------
* Syntax: ``scrapy genspider [-t template] <name> <domain>``
* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
* Requires project: *no*
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. versionadded:: 2.6.0
The ability to pass a URL instead of a domain.
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. note:: Even if an HTTPS URL is specified, the protocol used in
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
Usage example::
@ -265,11 +271,31 @@ crawl
Start crawling using a spider.
Supported options:
* ``-h, --help``: show a help message and exit
* ``-a NAME=VALUE``: set a spider argument (may be repeated)
* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
Usage examples::
$ scrapy crawl myspider
[ ... myspider starts crawling ... ]
$ scrapy -o myfile:csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
$ scrapy -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
$ scrapy -o myfile -t csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check
@ -598,8 +624,6 @@ Example:
Register commands via setup.py entry points
-------------------------------------------
.. note:: This is an experimental feature, use with caution.
You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.

84
docs/topics/components.rst Normal file
View File

@ -0,0 +1,84 @@
.. _topics-components:
==========
Components
==========
A Scrapy component is any class whose objects are created using
:func:`scrapy.utils.misc.create_instance`.
That includes the classes that you may assign to the following settings:
- :setting:`DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings <topics-settings>`, to
modify their behavior.
.. _enforce-component-requirements:
Enforcing component requirements
================================
Sometimes, your components may only be intended to work under certain
conditions. For example, they may require a minimum version of Scrapy to work as
intended, or they may require certain settings to have specific values.
In addition to describing those conditions in the documentation of your
component, it is a good practice to raise an exception from the ``__init__``
method of your component if those conditions are not met at run time.
In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
:ref:`extensions <topics-extensions>`, :ref:`item pipelines
<topics-item-pipeline>`, and :ref:`spider middlewares
<topics-spider-middleware>`, you should raise
:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
parameter to the exception so that it is printed in the logs, for the user to
see. For other components, feel free to raise whatever other exception feels
right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
version mismatch, while :exc:`ValueError` may be better if the issue is the
value of a setting.
If your requirement is a minimum Scrapy version, you may use
:attr:`scrapy.__version__` to enforce your requirement. For example::
from pkg_resources import parse_version
import scrapy
class MyComponent:
def __init__(self):
if parse_version(scrapy.__version__) < parse_version('2.7'):
raise RuntimeError(
f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
f"later, which allow defining the process_spider_output "
f"method of spider middlewares as an asynchronous "
f"generator."
)

6
docs/topics/contracts.rst
View File

@ -37,7 +37,7 @@ This callback is tested using three built-in contracts:
.. class:: CallbackKeywordArgumentsContract
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
attribute for the sample request. It must be a valid JSON dictionary.
::
@ -88,7 +88,7 @@ override three methods:
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
for request object. :class:`~scrapy.http.Request` is used by default,
for request object. :class:`~scrapy.Request` is used by default,
but this can be changed with the ``request_cls`` attribute.
If multiple contracts in chain have this attribute defined, the last one is used.
@ -102,7 +102,7 @@ override three methods:
.. method:: Contract.post_process(output)
This allows processing the output of the callback. Iterators are
converted listified before being passed to this hook.
converted to lists before being passed to this hook.
Raise :class:`~scrapy.exceptions.ContractFail` from
:class:`~scrapy.contracts.Contract.pre_process` or

150
docs/topics/coroutines.rst
View File

@ -1,3 +1,5 @@
.. _topics-coroutines:
==========
Coroutines
==========
@ -15,16 +17,14 @@ Supported callables
The following callables may be defined as coroutines using ``async def``, and
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.http.Request` callbacks.
- :class:`~scrapy.Request` callbacks.
.. note:: The callback output is not processed until the whole callback
finishes.
If you are using any custom or third-party :ref:`spider middleware
<topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
As a side effect, if the callback raises an exception, none of its
output is processed.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
.. versionchanged:: 2.7
Output of async callbacks is now processed asynchronously instead of
collecting all of it first.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -39,12 +39,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
Usage
=====
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method of :ref:`spider middlewares <topics-spider-middleware>`.
There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
It must be defined as an :term:`asynchronous generator`. The input
``result`` parameter is an :term:`asynchronous iterable`.
See also :ref:`sync-async-spider-middleware` and
:ref:`universal-spider-middleware`.
.. versionadded:: 2.7
General usage
=============
There are several use cases for coroutines in Scrapy.
Code that would return Deferreds when written for previous Scrapy versions,
such as downloader middlewares and signal handlers, can be rewritten to be
shorter and cleaner::
from itemadapter import ItemAdapter
@ -75,23 +89,28 @@ coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
class MySpider(Spider):
class MySpiderDeferred(Spider):
# ...
async def parse_with_deferred(self, response):
async def parse(self, response):
additional_response = await treq.get('https://additional.url')
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
async def parse_with_asyncio(self, response):
class MySpiderAsyncio(Spider):
# ...
async def parse(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
additional_data = await r.text()
additional_data = await additional_response.text()
# ... use response and additional_data to yield items and requests
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
:mod:`asyncio` loop and to use them you need to
:doc:`enable asyncio support in Scrapy<asyncio>`.
.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
you need to :ref:`wrap them<asyncio-await-dfd>`.
Common use cases for asynchronous code include:
* requesting data from websites, databases and other services (in callbacks,
@ -99,7 +118,100 @@ Common use cases for asynchronous code include:
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
(see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs
.. _sync-async-spider-middleware:
Mixing synchronous and asynchronous spider middlewares
======================================================
.. versionadded:: 2.7
The output of a :class:`~scrapy.Request` callback is passed as the ``result``
parameter to the
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
of the first :ref:`spider middleware <topics-spider-middleware>` from the
:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
Then the output of that ``process_spider_output`` method is passed to the
``process_spider_output`` method of the next spider middleware, and so on for
every active spider middleware.
Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
in this chain of calls.
However, if any of the ``process_spider_output`` methods is defined as a
synchronous method, and the previous ``Request`` callback or
``process_spider_output`` method is a coroutine, there are some drawbacks to
the asynchronous-to-synchronous conversion that Scrapy does so that the
synchronous ``process_spider_output`` method gets a synchronous iterable as its
``result`` parameter:
- The whole output of the previous ``Request`` callback or
``process_spider_output`` method is awaited at this point.
- If an exception raises while awaiting the output of the previous
``Request`` callback or ``process_spider_output`` method, none of that
output will be processed.
This contrasts with the regular behavior, where all items yielded before
an exception raises are processed.
Asynchronous-to-synchronous conversions are supported for backward
compatibility, but they are deprecated and will stop working in a future
version of Scrapy.
To avoid asynchronous-to-synchronous conversions, when defining ``Request``
callbacks as coroutine methods or when using spider middlewares whose
``process_spider_output`` method is an :term:`asynchronous generator`, all
active spider middlewares must either have their ``process_spider_output``
method defined as an asynchronous generator or :ref:`define a
process_spider_output_async method <universal-spider-middleware>`.
.. note:: When using third-party spider middlewares that only define a
synchronous ``process_spider_output`` method, consider
:ref:`making them universal <universal-spider-middleware>` through
:ref:`subclassing <tut-inheritance>`.
.. _universal-spider-middleware:
Universal spider middlewares
============================
.. versionadded:: 2.7
To allow writing a spider middleware that supports asynchronous execution of
its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
while maintaining support for older Scrapy versions, you may define
``process_spider_output`` as a synchronous method and define an
:term:`asynchronous generator` version of that method with an alternative name:
``process_spider_output_async``.
For example::
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
# ... do something with r
yield r
async def process_spider_output_async(self, response, result, spider):
async for r in result:
# ... do something with r
yield r
.. note:: This is an interim measure to allow, for a time, to write code that
works in Scrapy 2.7 and later without requiring
asynchronous-to-synchronous conversions, and works in earlier Scrapy
versions as well.
In some future version of Scrapy, however, this feature will be
deprecated and, eventually, in a later version of Scrapy, this
feature will be removed, and all spider middlewares will be expected
to define their ``process_spider_output`` method as an asynchronous
generator.

2
docs/topics/debug.rst
View File

@ -36,7 +36,7 @@ Consider the following Scrapy spider below::
Basically this is a simple spider which parses two pages of items (the
start_urls). Items also have a details page with additional information, so we
use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a
partially populated item.

32
docs/topics/deploy.rst
View File

@ -14,7 +14,7 @@ spiders come in.
Popular choices for deploying Scrapy spiders are:
* :ref:`Scrapyd <deploy-scrapyd>` (open source)
* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
* :ref:`Zyte Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
.. _deploy-scrapyd:
@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers.
.. _deploy-scrapy-cloud:
Deploying to Scrapy Cloud
=========================
Deploying to Zyte Scrapy Cloud
==============================
`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
the company behind Scrapy.
`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company
behind Scrapy.
Scrapy Cloud removes the need to setup and monitor servers
and provides a nice UI to manage spiders and review scraped items,
logs and stats.
Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a
nice UI to manage spiders and review scraped items, logs and stats.
To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
Please refer to the `Scrapy Cloud documentation`_ for more information.
To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line
tool.
Please refer to the `Zyte Scrapy Cloud documentation`_ for more information.
Scrapy Cloud is compatible with Scrapyd and one can switch between
Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between
them as needed - the configuration is read from the ``scrapy.cfg`` file
just like ``scrapyd-deploy``.
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
.. _shub: https://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: https://scrapinghub.com/
.. _shub: https://shub.readthedocs.io/en/latest/
.. _Zyte: https://zyte.com/
.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/
.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html

26
docs/topics/developer-tools.rst
View File

@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
Since Developer Tools operate on a live browser DOM, what you'll actually see
when inspecting the page source is not the original HTML, but a modified one
after applying some browser clean up and executing Javascript code. Firefox,
after applying some browser clean up and executing JavaScript code. Firefox,
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
the other hand, does not modify the original page HTML, so you won't be able to
extract any data if you use ``<tbody>`` in your XPath expressions.
Therefore, you should keep in mind the following things:
* Disable Javascript while inspecting the DOM looking for XPaths to be
* Disable JavaScript while inspecting the DOM looking for XPaths to be
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
* Never use full XPath paths, use relative and clever ones based on attributes
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
copy XPaths to selected elements. Let's try it out.
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
$ scrapy shell "https://quotes.toscrape.com/"
Then, back to your web browser, right-click on the ``span`` tag, select
``Copy > XPath`` and paste it in the Scrapy shell like so:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
page = 1
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
def parse(self, response):
data = json.loads(response.text)
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
@ -274,13 +274,13 @@ In more complex websites, it could be difficult to easily reproduce the
requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
format, by right-clicking on each of them in the network tool and using the
:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
request::
from scrapy import Request
request = Request.from_curl(
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
.. _quotes.toscrape.com: http://quotes.toscrape.com
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
.. _quotes.toscrape.com: https://quotes.toscrape.com
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions

118
docs/topics/downloader-middleware.rst
View File

@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
middleware.
:meth:`process_request` should either: return ``None``, return a
:class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request`
:class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request`
object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`.
If it returns ``None``, Scrapy will continue processing this request, executing all
@ -88,8 +88,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
or the appropriate download function; it'll return that response. The :meth:`process_response`
methods of installed middleware is always called on every response.
If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling
process_request methods and reschedule the returned request. Once the newly returned
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
:meth:`process_request` methods and reschedule the returned request. Once the newly returned
request is performed, the appropriate middleware chain will be called on
the downloaded response.
@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
ignored and not logged (unlike other exceptions).
:param request: the request being processed
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_response(request, response, spider)
:meth:`process_response` should either: return a :class:`~scrapy.http.Response`
object, return a :class:`~scrapy.http.Request` object or
object, return a :class:`~scrapy.Request` object or
raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception.
If it returns a :class:`~scrapy.http.Response` (it could be the same given
response, or a brand-new one), that response will continue to be processed
with the :meth:`process_response` of the next middleware in the chain.
If it returns a :class:`~scrapy.http.Request` object, the middleware chain is
If it returns a :class:`~scrapy.Request` object, the middleware chain is
halted and the returned request is rescheduled to be downloaded in the future.
This is the same behavior as if a request is returned from :meth:`process_request`.
@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
exception, it is ignored and not logged (unlike other exceptions).
:param request: the request that originated the response
:type request: is a :class:`~scrapy.http.Request` object
:type request: is a :class:`~scrapy.Request` object
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_exception(request, exception, spider)
@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception)
:meth:`process_exception` should return: either ``None``,
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object.
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_exception` methods of installed middleware,
@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
method chain of installed middleware is started, and Scrapy won't bother calling
any other :meth:`process_exception` methods of middleware.
If it returns a :class:`~scrapy.http.Request` object, the returned request is
If it returns a :class:`~scrapy.Request` object, the returned request is
rescheduled to be downloaded in the future. This stops the execution of
:meth:`process_exception` methods of the middleware the same as returning a
response would.
:param request: the request that generated the exception
:type request: is a :class:`~scrapy.http.Request` object
:type request: is a :class:`~scrapy.Request` object
:param exception: the raised exception
:type exception: an ``Exception`` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
@ -203,10 +203,15 @@ CookiesMiddleware
browsers do.
.. caution:: When non-UTF8 encoded byte sequences are passed to a
:class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
:class:`~scrapy.Request`, the ``CookiesMiddleware`` will log
a warning. Refer to :ref:`topics-logging-advanced-customization`
to customize the logging behaviour.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
The following settings can be used to configure the cookie middleware:
* :setting:`COOKIES_ENABLED`
@ -253,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will
**not** be merged with the existing cookies.
For more detailed information see the ``cookies`` parameter in
:class:`~scrapy.http.Request`.
:class:`~scrapy.Request`.
.. setting:: COOKIES_DEBUG
@ -318,8 +323,21 @@ HttpAuthMiddleware
This middleware authenticates all requests generated from certain spiders
using `Basic access authentication`_ (aka. HTTP auth).
To enable HTTP authentication from certain spiders, set the ``http_user``
and ``http_pass`` attributes of those spiders.
To enable HTTP authentication for a spider, set the ``http_user`` and
``http_pass`` spider attributes to the authentication data and the
``http_auth_domain`` spider attribute to the domain which requires this
authentication (its subdomains will be also handled in the same way).
You can set ``http_auth_domain`` to ``None`` to enable the
authentication for all requests but you risk leaking your authentication
credentials to unrelated domains.
.. warning::
In previous Scrapy versions HttpAuthMiddleware sent the authentication
data with all requests, which is a security problem if the spider
makes requests to several different domains. Currently if the
``http_auth_domain`` attribute is not set, the middleware will use the
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example::
@ -329,6 +347,7 @@ HttpAuthMiddleware
http_user = 'someuser'
http_pass = 'somepass'
http_auth_domain = 'intranet.example.com'
name = 'intranet.example.com'
# .. rest of the spider code omitted ...
@ -347,7 +366,7 @@ HttpCacheMiddleware
This middleware provides low-level cache to all HTTP requests and responses.
It has to be combined with a cache storage backend as well as a cache policy.
Scrapy ships with three HTTP cache storage backends:
Scrapy ships with the following HTTP cache storage backends:
* :ref:`httpcache-storage-fs`
* :ref:`httpcache-storage-dbm`
@ -496,7 +515,7 @@ defines the methods described below.
the :signal:`open_spider <spider_opened>` signal.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: close_spider(spider)
@ -504,27 +523,27 @@ defines the methods described below.
the :signal:`close_spider <spider_closed>` signal.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: retrieve_response(spider, request)
Return response if present in cache, or ``None`` otherwise.
:param spider: the spider which generated the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param request: the request to find cached response for
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
.. method:: store_response(spider, request, response)
Store the given response in the cache.
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param request: the corresponding request the spider generated
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param response: the response to store in the cache
:type response: :class:`~scrapy.http.Response` object
@ -684,11 +703,15 @@ HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ responses,
provided `brotlipy`_ is installed.
This middleware also supports decoding `brotli-compressed`_ as well as
`zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.org/project/brotlipy/
.. _brotli: https://pypi.org/project/Brotli/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -714,7 +737,7 @@ HttpProxyMiddleware
.. class:: HttpProxyMiddleware
This middleware sets the HTTP proxy to use for requests, by setting the
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
``proxy`` meta value for :class:`~scrapy.Request` objects.
Like the Python standard library module :mod:`urllib.request`, it obeys
the following environment variables:
@ -741,12 +764,12 @@ RedirectMiddleware
.. reqmeta:: redirect_urls
The urls which the request goes through (while being redirected) can be found
in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
in the ``redirect_urls`` :attr:`Request.meta <scrapy.Request.meta>` key.
.. reqmeta:: redirect_reasons
The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
``redirect_reasons`` :attr:`Request.meta <scrapy.Request.meta>` key. For
example: ``[301, 302, 307, 'meta refresh']``.
The format of a reason depends on the middleware that handled the corresponding
@ -762,7 +785,7 @@ settings (see the settings documentation for more info):
.. reqmeta:: dont_redirect
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_redirect``
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_redirect``
key set to True, the request will be ignored by this middleware.
If you want to handle some redirect status codes in your spider, you can
@ -775,7 +798,7 @@ responses (and pass them through to your spider) you can do this::
handle_httpstatus_list = [301, 302]
The ``handle_httpstatus_list`` key of :attr:`Request.meta
<scrapy.http.Request.meta>` can also be used to specify which response codes to
<scrapy.Request.meta>` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key
``handle_httpstatus_all`` to ``True`` if you want to allow any response code
for a request.
@ -881,9 +904,14 @@ settings (see the settings documentation for more info):
.. reqmeta:: dont_retry
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
set to True, the request will be ignored by this middleware.
To retry requests from a spider callback, you can use the
:func:`get_retry_request` function:
.. autofunction:: get_retry_request
RetryMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~
@ -906,7 +934,7 @@ Default: ``2``
Maximum number of times to retry, in addition to the first download.
Maximum number of retries can also be specified per-request using
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.Request.meta>`.
When initialized, the :reqmeta:`max_retry_times` meta key takes higher
precedence over the :setting:`RETRY_TIMES` setting.
@ -924,6 +952,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
^^^^^^^^^^^^^^^^^^^^^
Default: ``-1``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. _topics-dlmw-robots:
@ -961,7 +1001,7 @@ RobotsTxtMiddleware
.. reqmeta:: dont_obey_robotstxt
If :attr:`Request.meta <scrapy.http.Request.meta>` has
If :attr:`Request.meta <scrapy.Request.meta>` has
``dont_obey_robotstxt`` key set to True
the request will be ignored by this middleware even if
:setting:`ROBOTSTXT_OBEY` is enabled.
@ -980,7 +1020,7 @@ Parsers vary in several aspects:
(shorter) rule
Performance comparison of different parsers is available at `the following link
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
<https://github.com/scrapy/scrapy/issues/3969>`_.
.. _protego-parser:
@ -1045,9 +1085,13 @@ In order to use this parser:
* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
.. warning:: `Upstream issue #122
<https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
* Set :setting:`ROBOTSTXT_PARSER` setting to
``scrapy.robotstxt.ReppyRobotParser``
.. _rerp-parser:
Robotexclusionrulesparser
@ -1075,7 +1119,7 @@ In order to use this parser:
.. _support-for-new-robots-parser:
Implementing support for a new parser
-------------------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can implement support for a new robots.txt_ parser by subclassing
the abstract base class :class:`~scrapy.robotstxt.RobotParser` and

54
docs/topics/dynamic-content.rst
View File

@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the
information can be found in the response they get.
If they get a response with the desired data, modify your Scrapy
:class:`~scrapy.http.Request` to match that of the other HTTP client. For
:class:`~scrapy.Request` to match that of the other HTTP client. For
example, try using the same user-agent string (:setting:`USER_AGENT`) or the
same :attr:`~scrapy.http.Request.headers`.
same :attr:`~scrapy.Request.headers`.
If they also get a response without the desired data, you’ll need to take
steps to make your request more similar to that of the web browser. See
@ -81,14 +81,14 @@ Use the :ref:`network tool <topics-network-tool>` of your web browser to see
how your web browser performs the desired request, and try to reproduce that
request with Scrapy.
It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
method and URL. However, you may also need to reproduce the body, headers and
form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
form parameters (see :class:`~scrapy.FormRequest`) of that request.
As all major browsers allow to export the requests in `cURL
<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
:class:`~scrapy.http.Request` from a cURL command. To get more information
:meth:`~scrapy.Request.from_curl()` to generate an equivalent
:class:`~scrapy.Request` from a cURL command. To get more information
visit :ref:`request from curl <requests-from-curl>` inside the network
tool section.
@ -125,7 +125,7 @@ data from it depends on the type of response:
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a
:class:`~scrapy.selector.Selector` and then
:class:`~scrapy.Selector` and then
:ref:`use it <topics-selectors>` as usual::
selector = Selector(data['html'])
@ -246,24 +246,46 @@ Using a headless browser
========================
A `headless browser`_ is a special web browser that provides an API for
automation.
automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
The easiest way to use a headless browser with Scrapy is to use Selenium_,
along with `scrapy-selenium`_ for seamless integration.
One such library is `playwright-python`_ (an official Python port of `playwright`_).
The following is a simple snippet to illustrate its usage within a Scrapy spider::
import scrapy
from playwright.async_api import async_playwright
class PlaywrightSpider(scrapy.Spider):
name = "playwright"
start_urls = ["data:,"] # avoid using the default Scrapy downloader
async def parse(self, response):
async with async_playwright() as pw:
browser = await pw.chromium.launch()
page = await browser.new_page()
await page.goto("https:/example.org")
title = await page.title()
return {"title": title}
However, using `playwright-python`_ directly as in the above example
circumvents most of the Scrapy components (middlewares, dupefilter, etc).
We recommend using `scrapy-playwright`_ for a better integration.
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
.. _chompjs: https://github.com/Nykakin/chompjs
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _Splash: https://github.com/scrapinghub/splash
.. _chompjs: https://github.com/Nykakin/chompjs
.. _curl: https://curl.haxx.se/
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _js2xml: https://github.com/scrapinghub/js2xml
.. _playwright-python: https://github.com/microsoft/playwright-python
.. _playwright: https://github.com/microsoft/playwright
.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
.. _pytesseract: https://github.com/madmaze/pytesseract
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _Selenium: https://www.selenium.dev/
.. _Splash: https://github.com/scrapinghub/splash
.. _tabula-py: https://github.com/chezou/tabula-py
.. _wget: https://www.gnu.org/software/wget/
.. _wgrep: https://github.com/stav/wgrep
.. _wgrep: https://github.com/stav/wgrep

15
docs/topics/exceptions.rst
View File

@ -64,10 +64,10 @@ NotConfigured
This exception can be raised by some components to indicate that they will
remain disabled. Those components include:
* Extensions
* Item pipelines
* Downloader middlewares
* Spider middlewares
- Extensions
- Item pipelines
- Downloader middlewares
- Spider middlewares
The exception must be raised in the component's ``__init__`` method.
@ -85,8 +85,8 @@ StopDownload
.. exception:: StopDownload(fail=True)
Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
indicate that no further bytes should be downloaded for a response.
Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signal handler to indicate that no further bytes should be downloaded for a response.
The ``fail`` boolean parameter controls which method will handle the resulting
response:
@ -110,5 +110,6 @@ attribute.
``StopDownload(False)`` or ``StopDownload(True)`` will raise
a :class:`TypeError`.
See the documentation for the :class:`~scrapy.signals.bytes_received` signal
See the documentation for the :class:`~scrapy.signals.bytes_received` and
:class:`~scrapy.signals.headers_received` signals
and the :ref:`topics-stop-response-download` topic for additional information and examples.

51
docs/topics/exporters.rst
View File

@ -50,18 +50,19 @@ value of one of their fields::
self.year_to_exporter = {}
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
for exporter, xml_file in self.year_to_exporter.values():
exporter.finish_exporting()
xml_file.close()
def _exporter_for_item(self, item):
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(f)
xml_file = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(xml_file)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
return self.year_to_exporter[year]
self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0]
def process_item(self, item, spider):
exporter = self._exporter_for_item(item)
@ -89,7 +90,7 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
If you use :class:`~.Item` you can declare a serializer in the
If you use :class:`~scrapy.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
@ -116,14 +117,14 @@ after your custom code.
Example::
from scrapy.exporter import XmlItemExporter
from scrapy.exporters import XmlItemExporter
class ProductXmlExporter(XmlItemExporter):
def serialize_field(self, field, name, value):
if field == 'price':
if name == 'price':
return f'$ {str(value)}'
return super(Product, self).serialize_field(field, name, value)
return super().serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -171,7 +172,7 @@ BaseItemExporter
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
:class:`dict`.
:type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
:type field: :class:`~scrapy.Field` object or a :class:`dict` instance
:param name: the name of the field being serialized
:type name: str
@ -194,17 +195,25 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or ``None`` if
you want to export all fields. Defaults to ``None``.
Fields to export, their order [1]_ and their output names.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Possible values are:
When using :ref:`item objects <item-types>` that do not expose all their
possible fields, exporters that do not support exporting a different
subset of fields per item will only export the fields found in the first
item exported. Use ``fields_to_export`` to define all the fields to be
exported.
- ``None`` (all fields [2]_, default)
- A list of fields::
['field1', 'field2']
- A dict where keys are fields and values are output names::
{'field1': 'Field 1', 'field2': 'Field 2'}
.. [1] Not all exporters respect the specified field order.
.. [2] When using :ref:`item objects <item-types>` that do not expose
all their possible fields, exporters that do not support exporting
a different subset of fields per item will only export the fields
found in the first item exported.
.. attribute:: export_empty_fields
@ -296,8 +305,8 @@ CsvItemExporter
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
CSV columns, their order and their column names. The
:attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)

22
docs/topics/extensions.rst
View File

@ -7,8 +7,7 @@ Extensions
The extensions framework provides a mechanism for inserting your own
custom functionality into Scrapy.
Extensions are just regular classes that are instantiated at Scrapy startup,
when extensions are initialized.
Extensions are just regular classes.
Extension settings
==================
@ -18,7 +17,7 @@ settings, just like any other Scrapy code.
It is customary for extensions to prefix their settings with their own name, to
avoid collision with existing (and future) extensions. For example, a
hypothetic extension to handle `Google Sitemaps`_ would use settings like
hypothetical extension to handle `Google Sitemaps`_ would use settings like
``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
@ -27,8 +26,8 @@ Loading & activating extensions
===============================
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
code must be performed in the class ``__init__`` method.
instance of the extension class per spider being run. All the extension
initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@ -257,6 +256,12 @@ settings:
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
.. note::
When a certain closing condition is met, requests which are
currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS`
requests) are still processed.
.. setting:: CLOSESPIDER_TIMEOUT
CLOSESPIDER_TIMEOUT
@ -279,8 +284,6 @@ Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``.
Requests which are currently in the downloader queue (up to
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
If zero (or non set), spiders won't be closed by number of passed items.
.. setting:: CLOSESPIDER_PAGECOUNT
@ -320,6 +323,11 @@ domain has finished scraping, including the Scrapy stats collected. The email
will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
setting.
Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a
full list of parameters, including examples on how to instantiate
:class:`~scrapy.mail.MailSender` and use mail settings, see
:ref:`topics-email`.
.. module:: scrapy.extensions.debug
:synopsis: Extensions for debugging Scrapy

317
docs/topics/feed-exports.rst
View File

@ -21,10 +21,10 @@ Serialization formats
For serializing the scraped data, the feed exports use the :ref:`Item exporters
<topics-exporters>`. These formats are supported out of the box:
* :ref:`topics-feed-format-json`
* :ref:`topics-feed-format-jsonlines`
* :ref:`topics-feed-format-csv`
* :ref:`topics-feed-format-xml`
- :ref:`topics-feed-format-json`
- :ref:`topics-feed-format-jsonlines`
- :ref:`topics-feed-format-csv`
- :ref:`topics-feed-format-xml`
But you can also extend the supported format through the
:setting:`FEED_EXPORTERS` setting.
@ -34,54 +34,58 @@ But you can also extend the supported format through the
JSON
----
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
- Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
- See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
.. _topics-feed-format-jsonlines:
JSON lines
----------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
.. _topics-feed-format-csv:
CSV
---
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
.. _topics-feed-format-xml:
XML
---
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
- Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
.. _topics-feed-format-pickle:
Pickle
------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
- Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
.. _topics-feed-format-marshal:
Marshal
-------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
.. _topics-feed-storage:
@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
* :ref:`topics-feed-storage-stdout`
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
@ -114,8 +118,8 @@ Storage URI parameters
The storage URI can also contain parameters that get replaced when the feed is
being created. These parameters are:
* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
* ``%(name)s`` - gets replaced by the spider name
- ``%(time)s`` - gets replaced by a timestamp when the feed is being created
- ``%(name)s`` - gets replaced by the spider name
Any other named parameter gets replaced by the spider attribute of the same
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
@ -123,13 +127,16 @@ attribute the moment the feed is being created.
Here are some examples to illustrate:
* Store in FTP using one directory per spider:
- Store in FTP using one directory per spider:
* ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
- ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
* Store in S3 using one directory per spider:
- Store in S3 using one directory per spider:
* ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
they can also be used as storage URI parameters.
.. _topics-feed-storage-backends:
@ -144,9 +151,9 @@ Local filesystem
The feeds are stored in the local filesystem.
* URI scheme: ``file``
* Example URI: ``file:///tmp/export.csv``
* Required external libraries: none
- URI scheme: ``file``
- Example URI: ``file:///tmp/export.csv``
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
@ -159,9 +166,9 @@ FTP
The feeds are stored in a FTP server.
* URI scheme: ``ftp``
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
- URI scheme: ``ftp``
- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
- Required external libraries: none
FTP supports two different connection modes: `active or passive
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
@ -178,23 +185,29 @@ S3
The feeds are stored on `Amazon S3`_.
* URI scheme: ``s3``
* Example URIs:
- URI scheme: ``s3``
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Example URIs:
* Required external libraries: `botocore`_
- ``s3://mybucket/path/to/export.csv``
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_)
You can also define a custom ACL for exported feeds using this setting:
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
* :setting:`FEED_STORAGE_S3_ACL`
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -208,19 +221,20 @@ Google Cloud Storage (GCS)
The feeds are stored on `Google Cloud Storage`_.
* URI scheme: ``gs``
* Example URIs:
- URI scheme: ``gs``
* ``gs://mybucket/path/to/export.csv``
- Example URIs:
* Required external libraries: `google-cloud-storage`_.
- ``gs://mybucket/path/to/export.csv``
- Required external libraries: `google-cloud-storage`_.
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
* :setting:`FEED_STORAGE_GCS_ACL`
* :setting:`GCS_PROJECT_ID`
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -234,9 +248,9 @@ Standard output
The feeds are written to the standard output of the Scrapy process.
* URI scheme: ``stdout``
* Example URI: ``stdout:``
* Required external libraries: none
- URI scheme: ``stdout``
- Example URI: ``stdout:``
- Required external libraries: none
.. _delayed-file-delivery:
@ -259,21 +273,118 @@ soon as a file reaches the maximum item count, that file is delivered to the
feed URI, allowing item delivery to start way before the end of the crawl.
.. _item-filter:
Item filtering
==============
.. versionadded:: 2.6.0
You can filter items that you want to allow for a particular feed by using the
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
the specified types will be added to the feed.
The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`
class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
method ``accepts`` and taking ``feed_options`` as an argument.
For instance::
class MyCustomFilter:
def __init__(self, feed_options):
self.feed_options = feed_options
def accepts(self, item):
if "field1" in item and item["field1"] == "expected_data":
return True
return False
You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.
See :setting:`FEEDS` for examples.
ItemFilter
----------
.. autoclass:: scrapy.extensions.feedexport.ItemFilter
:members:
.. _post-processing:
Post-Processing
===============
.. versionadded:: 2.6.0
Scrapy provides an option to activate plugins to post-process feeds before they are exported
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
can create your own :ref:`plugins <custom-plugins>`.
These plugins can be activated through the ``postprocessing`` option of a feed.
The option must be passed a list of post-processing plugins in the order you want
the feed to be processed. These plugins can be declared either as an import string
or with the imported class of the plugin. Parameters to plugins can be passed
through the feed options. See :ref:`feed options <feed-options>` for examples.
.. _builtin-plugins:
Built-in Plugins
----------------
.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
.. _custom-plugins:
Custom Plugins
--------------
Each plugin is a class that must implement the following methods:
.. method:: __init__(self, file, feed_options)
Initialize the plugin.
:param file: file-like object having at least the `write`, `tell` and `close` methods implemented
:param feed_options: feed-specific :ref:`options <feed-options>`
:type feed_options: :class:`dict`
.. method:: write(self, data)
Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
It must return number of bytes written.
.. method:: close(self)
Close the target file object.
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
can then access those parameters from the ``__init__`` method of your plugin.
Settings
========
These are the settings used for configuring the feed exports:
* :setting:`FEEDS` (mandatory)
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
- :setting:`FEEDS` (mandatory)
- :setting:`FEED_EXPORT_ENCODING`
- :setting:`FEED_STORE_EMPTY`
- :setting:`FEED_EXPORT_FIELDS`
- :setting:`FEED_EXPORT_INDENT`
- :setting:`FEED_STORAGES`
- :setting:`FEED_STORAGE_FTP_ACTIVE`
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport
@ -301,21 +412,31 @@ For instance::
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
'fields': None,
'indent': 4,
},
'item_export_kwargs': {
'export_empty_fields': True,
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],
'item_filter': MyCustomFilter1,
'encoding': 'latin1',
'indent': 8,
},
pathlib.Path('items.csv'): {
pathlib.Path('items.csv.gz'): {
'format': 'csv',
'fields': ['price', 'name'],
'item_filter': 'myproject.filters.MyCustomFilter2',
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
'gzip_compresslevel': 5,
},
}
.. _feed-options:
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition:
@ -326,12 +447,30 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
- ``item_classes``: list of :ref:`item classes <topics-items>` to export.
If undefined or empty, all items are exported.
.. versionadded:: 2.6.0
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
.. versionadded:: 2.6.0
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
.. versionadded:: 2.4.0
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
@ -350,10 +489,17 @@ as a fallback value if that key is not provided for a specific feed definition:
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
.. versionadded:: 2.4.0
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
- ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
The plugins will be used in the order of the list passed.
.. versionadded:: 2.6.0
.. setting:: FEED_EXPORT_ENCODING
@ -376,18 +522,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
A list of fields to export, optional.
Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
defined in :ref:`item objects <topics-items>` yielded by your spider.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
order and their output names. See :attr:`BaseItemExporter.fields_to_export
<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
.. setting:: FEED_EXPORT_INDENT
@ -492,6 +629,7 @@ Default::
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@ -512,7 +650,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
-----------------------------
----------------------------
.. versionadded:: 2.3.0
Default: ``0``
@ -581,18 +721,25 @@ The function signature should be as follows:
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
is always ``1``.
.. versionadded:: 2.3.0
- ``batch_time``: UTC date and time, in ISO format with ``:``
replaced with ``-``.
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``time``: ``batch_time``, with microseconds set to ``0``.
:type params: dict
:param spider: source spider of the feed items
:type spider: scrapy.spiders.Spider
:type spider: scrapy.Spider
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
.. caution:: The function should return a new dictionary, modifying
the received ``params`` in-place is deprecated.
For example, to include the :attr:`name <scrapy.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
@ -608,7 +755,7 @@ source spider in the feed URI:
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier

22
docs/topics/item-pipeline.rst
View File

@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
Additionally, they may also implement the following methods:
@ -51,18 +51,18 @@ Additionally, they may also implement the following methods:
This method is called when the spider is opened.
:param spider: the spider which was opened
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: close_spider(self, spider)
This method is called when the spider is closed.
:param spider: the spider which was closed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
.. classmethod:: from_crawler(cls, crawler)
If present, this classmethod is called to create a pipeline instance
If present, this class method is called to create a pipeline instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the pipeline. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for pipeline to
@ -99,11 +99,11 @@ contain a price::
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
--------------------------
Write items to a JSON lines file
--------------------------------
The following pipeline stores all scraped items (from all spiders) into a
single ``items.jl`` file, containing one item per line serialized in JSON
single ``items.jsonl`` file, containing one item per line serialized in JSON
format::
import json
@ -113,7 +113,7 @@ format::
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'w')
self.file = open('items.jsonl', 'w')
def close_spider(self, spider):
self.file.close()
@ -190,6 +190,8 @@ item.
import scrapy
from itemadapter import ItemAdapter
from scrapy.utils.defer import maybe_deferred_to_future
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
@ -202,7 +204,7 @@ item.
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await spider.crawler.engine.download(request, spider)
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
if response.status != 200:
# Error happened, return item.

15
docs/topics/items.rst
View File

@ -42,7 +42,8 @@ Item objects
:class:`Item` provides a :class:`dict`-like API plus additional features that
make it the most feature-complete item type:
.. class:: Item([arg])
.. class:: scrapy.item.Item([arg])
.. class:: scrapy.Item([arg])
:class:`Item` objects replicate the standard :class:`dict` API, including
its ``__init__`` method.
@ -101,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
They work natively in Python 3.7 or later, or using the `dataclasses
backport`_ in Python 3.6.
.. _dataclasses backport: https://pypi.org/project/dataclasses/
Example::
from dataclasses import dataclass
@ -199,7 +195,8 @@ It's important to note that the :class:`Field` objects used to declare the item
do not stay assigned as class attributes. Instead, they can be accessed through
the :attr:`Item.fields` attribute.
.. class:: Field([arg])
.. class:: scrapy.item.Field([arg])
.. class:: scrapy.Field([arg])
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
doesn't provide any extra functionality or attributes. In other words,
@ -317,11 +314,11 @@ If that is not the desired behavior, use a deep copy instead.
See :mod:`copy` for more information.
To create a shallow copy of an item, you can either call
:meth:`~scrapy.item.Item.copy` on an existing item
:meth:`~scrapy.Item.copy` on an existing item
(``product2 = product.copy()``) or instantiate your item class from an existing
item (``product2 = Product(product)``).
To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead
(``product2 = product.deepcopy()``).

8
docs/topics/jobs.rst
View File

@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
.. _topics-keeping-persistent-state-between-batches:
Keeping persistent state between batches
========================================
@ -65,7 +67,7 @@ Cookies expiration
------------------
Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
scheduled may no longer work. This won't be an issue if your spider doesn't rely
on cookies.
@ -74,10 +76,10 @@ on cookies.
Request serialization
---------------------
For persistence to work, :class:`~scrapy.http.Request` objects must be
For persistence to work, :class:`~scrapy.Request` objects must be
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
values passed to their ``__init__`` method, which must be methods of the
running :class:`~scrapy.spiders.Spider` class.
running :class:`~scrapy.Spider` class.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.

18
docs/topics/leaks.rst
View File

@ -27,7 +27,7 @@ Common causes of memory leaks
It happens quite often (sometimes by accident, sometimes on purpose) that the
Scrapy developer passes objects referenced in Requests (for example, using the
:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta`
attributes or the request callback function) and that effectively bounds the
lifetime of those referenced objects to the lifetime of the Request. This is,
by far, the most common cause of memory leaks in Scrapy projects, and a quite
@ -48,9 +48,9 @@ Too Many Requests?
------------------
By default Scrapy keeps the request queue in memory; it includes
:class:`~scrapy.http.Request` objects and all objects
referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
and :attr:`~scrapy.http.Request.meta`).
:class:`~scrapy.Request` objects and all objects
referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs`
and :attr:`~scrapy.Request.meta`).
While not necessarily a leak, this can take a lot of memory. Enabling
:ref:`persistent job queue <topics-jobs>` could help keeping memory usage
in control.
@ -90,11 +90,11 @@ Which objects are tracked?
The objects tracked by ``trackrefs`` are all from these classes (and all its
subclasses):
* :class:`scrapy.http.Request`
* :class:`scrapy.Request`
* :class:`scrapy.http.Response`
* :class:`scrapy.item.Item`
* :class:`scrapy.selector.Selector`
* :class:`scrapy.spiders.Spider`
* :class:`scrapy.Item`
* :class:`scrapy.Selector`
* :class:`scrapy.Spider`
A real example
--------------
@ -154,7 +154,7 @@ Too many spiders?
If your project has too many spiders executed in parallel,
the output of :func:`prefs()` can be difficult to read.
For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclases). For
ignore a particular class (and all its subclasses). For
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider

21
docs/topics/link-extractors.rst
View File

@ -10,12 +10,19 @@ The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
list of matching :class:`~scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
def parse(self, response):
for link in self.link_extractor.extract_links(response):
yield Request(link.url, callback=self.parse)
.. _topics-link-extractors-ref:
@ -145,4 +152,12 @@ LxmlLinkExtractor
.. automethod:: extract_links
Link
----
.. module:: scrapy.link
:synopsis: Link from link extractors
.. autoclass:: Link
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

2
docs/topics/loaders.rst
View File

@ -56,7 +56,7 @@ chapter <topics-items>`::
l.add_xpath('name', '//div[@class="product_name"]')
l.add_xpath('name', '//div[@class="product_title"]')
l.add_xpath('price', '//p[@id="price"]')
l.add_css('stock', 'p#stock]')
l.add_css('stock', 'p#stock')
l.add_value('last_updated', 'today') # you can also use literal values
return l.load_item()

54
docs/topics/logging.rst
View File

@ -93,7 +93,7 @@ path::
Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
instance, which can be accessed and used like this::
import scrapy
@ -101,7 +101,7 @@ instance, which can be accessed and used like this::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapinghub.com']
start_urls = ['https://scrapy.org']
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
@ -117,7 +117,7 @@ Python logger you want. For example::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapinghub.com']
start_urls = ['https://scrapy.org']
def parse(self, response):
logger.info('Parse function called on %s', response.url)
@ -143,6 +143,7 @@ Logging settings
These settings can be used to configure the logging:
* :setting:`LOG_FILE`
* :setting:`LOG_FILE_APPEND`
* :setting:`LOG_ENABLED`
* :setting:`LOG_ENCODING`
* :setting:`LOG_LEVEL`
@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
:setting:`LOG_FILE` is set, messages sent through the root logger will be
redirected to a file named :setting:`LOG_FILE` with encoding
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
messages will be displayed on the standard error. Lastly, if
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
(discarding the output from previous runs, if any). Lastly, if
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many
HTTP 404 and 500 responses, and you want to hide all messages like this::
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
is not handled or not allowed
The first thing to note is a logger name - it is in brackets:
@ -242,6 +245,47 @@ e.g. in the spider's ``__init__`` method::
If you run this spider again then INFO messages from
``scrapy.spidermiddlewares.httperror`` logger will be gone.
You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================

152
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:
@ -56,6 +56,8 @@ this:
error will be logged and the file won't be present in the ``files`` field.
.. _images-pipeline:
Using the Images Pipeline
=========================
@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
JPEG/RGB format, so you need to install this library in order to use it.
`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
PIL.
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
.. _Python Imaging Library: http://www.pythonware.com/products/pil/
.. _topics-media-pipeline-enabling:
@ -113,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
IMAGES_STORE = '/path/to/valid/dir'
.. _topics-file-naming:
File Naming
===========
Default File Naming
-------------------
By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names.
For example, the following image URL::
http://www.example.com/image.jpg
Whose ``SHA-1 hash`` is::
3afec3b4765f8f0a07b78f98c07b83f013567a0a
Will be downloaded and stored using your chosen :ref:`storage method <topics-supported-storage>` and the following file name::
3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
Custom File Naming
-------------------
You may wish to use a different calculated file name for saved files.
For example, classifying an image by including meta in the file name.
Customize file names by overriding the ``file_path`` method of your
media pipeline.
For example, an image pipeline with image URL::
http://www.example.com/product/images/large/front/0000000004166
Can be processed into a file name with a condensed hash and the perspective
``front``::
00b08510e4_front.jpg
By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
image_perspective = request.url.split('/')[-2]
image_filename = f'{image_url_hash}_{image_perspective}.jpg'
return image_filename
.. warning::
If your custom file name scheme relies on meta data that can vary between
scrapes it may lead to unexpected re-downloading of existing media using
new file names.
For example, if your custom file name scheme uses a product title and the
site changes an item's product title between scrapes, Scrapy will re-download
the same media using updated file names.
For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`.
.. _topics-supported-storage:
Supported Storage
=================
File system storage
-------------------
The files are stored using a `SHA1 hash`_ of their URLs for the file names.
File system storage will save files to the following path::
For example, the following image URL::
http://www.example.com/image.jpg
Whose ``SHA1 hash`` is::
3afec3b4765f8f0a07b78f98c07b83f013567a0a
Will be downloaded and stored in the following file::
<IMAGES_STORE>/full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
<IMAGES_STORE>/full/<FILE_NAME>
Where:
@ -141,6 +196,9 @@ Where:
* ``full`` is a sub-directory to separate full images from thumbnails (if
used). For more info see :ref:`topics-images-thumbnails`.
* ``<FILE_NAME>`` is the file name assigned to the file. For more info see :ref:`topics-file-naming`.
.. _media-pipeline-ftp:
FTP server storage
@ -164,14 +222,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
the passive connection mode by default. To use the active connection mode instead,
set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
.. _media-pipelines-s3:
Amazon S3 storage
-----------------
.. setting:: FILES_STORE_S3_ACL
.. setting:: IMAGES_STORE_S3_ACL
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
bucket. Scrapy will automatically upload the files to the bucket.
If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
@ -187,8 +248,9 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@ -197,9 +259,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Minio: https://github.com/minio/minio
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _media-pipeline-gcs:
@ -256,7 +319,7 @@ respectively), the pipeline will put the results under the respective field
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
``images`` field. For instance, using the :class:`~scrapy.Item` class::
import scrapy
@ -293,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
Additional features
===================
.. _file-expiration:
File expiration
---------------
@ -320,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
and pipeline class MyPipeline will have expiration time set to 180.
The last modified time from the file is used to determine the age of the file in days,
which is then compared to the set expiration time to determine if the file is expired.
.. _topics-images-thumbnails:
Thumbnail generation for images
@ -350,9 +418,9 @@ Where:
* ``<size_name>`` is the one specified in the :setting:`IMAGES_THUMBS`
dictionary keys (``small``, ``big``, etc)
* ``<image_id>`` is the `SHA1 hash`_ of the image url
* ``<image_id>`` is the `SHA-1 hash`_ of the image url
.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
Example of image files stored using ``small`` and ``big`` thumbnail names::
@ -421,7 +489,7 @@ See here the methods that you can override in your custom Files Pipeline:
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
:class:`item <scrapy.Item>`
You can override this method to customize the download path of each file.
@ -446,6 +514,9 @@ See here the methods that you can override in your custom Files Pipeline:
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@ -557,7 +628,7 @@ See here the methods that you can override in your custom Images Pipeline:
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
:class:`item <scrapy.Item>`
You can override this method to customize the download path of each file.
@ -582,6 +653,29 @@ See here the methods that you can override in your custom Images Pipeline:
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
thumbnail download path of the image originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>`,
``thumb_id``,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.Item>`.
You can override this method to customize the thumbnail download path of each image.
You can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`thumb_path` method returns
``thumbs/<size name>/<request URL hash>.<extension>``.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,

43
docs/topics/practices.rst
View File

@ -63,7 +63,7 @@ project as example.
process = CrawlerProcess(get_project_settings())
# 'followall' is the name of one of the spiders of the project.
process.crawl('followall', domain='scrapinghub.com')
process.crawl('followall', domain='scrapy.org')
process.start() # the script will block here until the crawling is finished
There's another Scrapy utility that provides more control over the crawling
@ -119,6 +119,7 @@ Here is an example that runs multiple spiders simultaneously:
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously:
# Your second spider definition
...
process = CrawlerProcess()
settings = get_project_settings()
process = CrawlerProcess(settings)
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start() # the script will block here until all crawling jobs are finished
@ -141,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -151,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
...
configure_logging()
runner = CrawlerRunner()
settings = get_project_settings()
runner = CrawlerRunner(settings)
runner.crawl(MySpider1)
runner.crawl(MySpider2)
d = runner.join()
@ -166,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -175,8 +180,9 @@ Same example but running the spiders sequentially by chaining the deferreds:
# Your second spider definition
...
configure_logging()
runner = CrawlerRunner()
settings = get_project_settings()
configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
def crawl():
@ -187,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
crawl()
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some
limitations, to use these different values. What happens in practice is
different for different settings:
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
default one) cannot be read from the per-spider settings. These are applied
when the :class:`~scrapy.crawler.CrawlerRunner` or
:class:`~scrapy.crawler.CrawlerProcess` object is created.
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
available value is used, and if a spider requests a different reactor an
exception will be raised. These are applied when the reactor is installed.
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
the first available value is used. These are applied when the reactor is
started.
.. seealso:: :ref:`run-from-script`.
.. _distributed-crawls:
@ -237,14 +262,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
cookies to spot bot behaviour
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
directly
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
`Crawlera`_
`Zyte Smart Proxy Manager`_
If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
@ -252,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _Tor project: https://www.torproject.org/
.. _commercial support: https://scrapy.org/support/
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _Common Crawl: https://commoncrawl.org/
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

394
docs/topics/request-response.rst
View File

@ -26,10 +26,6 @@ Request objects
.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
a :class:`Response`.
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
@ -39,7 +35,7 @@ Request objects
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.spiders.Spider.parse` method will be used.
:meth:`~scrapy.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
:type callback: collections.abc.Callable
@ -61,6 +57,12 @@ Request objects
:param headers: the headers of this request. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers). If
``None`` is passed as value, the HTTP header will not be sent at all.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
:type headers: dict
:param cookies: the request cookies. These can be sent in two forms.
@ -90,7 +92,7 @@ Request objects
To create a request that does not send stored cookies and does not
store received cookies, set the ``dont_merge_cookies`` key to ``True``
in :attr:`request.meta <scrapy.http.Request.meta>`.
in :attr:`request.meta <scrapy.Request.meta>`.
Example of a request that sends manually-defined cookies and ignores
cookie storage::
@ -102,6 +104,16 @@ Request objects
)
For more info see :ref:`cookies-mw`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
.. versionadded:: 2.6.0
Cookie values that are :class:`bool`, :class:`float` or :class:`int`
are casted to :class:`str`.
:type cookies: dict or list
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
@ -193,6 +205,8 @@ Request objects
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. autoattribute:: Request.attributes
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
@ -208,6 +222,15 @@ Request objects
.. automethod:: from_curl
.. automethod:: to_dict
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.utils.request.request_from_dict
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@ -281,7 +304,7 @@ errors if needed::
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"http://www.httphttpbinbin.org/", # DNS error expected
"https://example.invalid/", # DNS error expected
]
def start_requests(self):
@ -316,6 +339,7 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
@ -341,6 +365,278 @@ achieve this by using ``Failure.request.cb_kwargs``::
main_url=failure.request.cb_kwargs['main_url'],
)
.. _request-fingerprints:
Request fingerprints
--------------------
There are some aspects of scraping, such as filtering out duplicate requests
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
unique identifier from a :class:`~scrapy.http.Request` object: a request
fingerprint.
You often do not need to worry about request fingerprints, the default request
fingerprinter works for most projects.
However, there is no universal way to generate a unique identifier from a
request, because different situations require comparing requests differently.
For example, sometimes you may need to compare URLs case-insensitively, include
URL fragments, exclude certain URL query parameters, include some or all
headers, etc.
To change how request fingerprints are built for your requests, use the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
.. setting:: REQUEST_FINGERPRINTER_CLASS
REQUEST_FINGERPRINTER_CLASS
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 2.7
Default: :class:`scrapy.utils.request.RequestFingerprinter`
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
import path.
.. autoclass:: scrapy.utils.request.RequestFingerprinter
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
REQUEST_FINGERPRINTER_IMPLEMENTATION
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 2.7
Default: ``'2.6'``
Determines which request fingerprinting algorithm is used by the default
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
Possible values are:
- ``'2.6'`` (default)
This implementation uses the same request fingerprinting algorithm as
Scrapy 2.6 and earlier versions.
Even though this is the default value for backward compatibility reasons,
it is a deprecated value.
- ``'2.7'``
This implementation was introduced in Scrapy 2.7 to fix an issue of the
previous implementation.
New projects should use this value. The :command:`startproject` command
sets this value in the generated ``settings.py`` file.
If you are using the default value (``'2.6'``) for this setting, and you are
using Scrapy components where changing the request fingerprinting algorithm
would cause undesired results, you need to carefully decide when to change the
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
setting to a custom request fingerprinter class that implements the 2.6 request
fingerprinting algorithm and does not log this warning (
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
class).
Scenarios where changing the request fingerprinting algorithm may cause
undesired results include, for example, using the HTTP cache middleware (see
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
Changing the request fingerprinting algorithm would invalidate the current
cache, requiring you to redownload all requests again.
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
your settings to switch already to the request fingerprinting implementation
that will be the only request fingerprinting implementation available in a
future version of Scrapy, and remove the deprecation warning triggered by using
the default value (``'2.6'``).
.. _2.6-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A request fingerprinter is a class that must implement the following method:
.. currentmodule:: None
.. method:: fingerprint(self, request)
Return a :class:`bytes` object that uniquely identifies *request*.
See also :ref:`request-fingerprint-restrictions`.
:param request: request to fingerprint
:type request: scrapy.http.Request
Additionally, it may also implement the following methods:
.. classmethod:: from_crawler(cls, crawler)
:noindex:
If present, this class method is called to create a request fingerprinter
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
new instance of the request fingerprinter.
*crawler* provides access to all Scrapy core components like settings and
signals; it is a way for the request fingerprinter to access them and hook
its functionality into Scrapy.
:param crawler: crawler that uses this request fingerprinter
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. classmethod:: from_settings(cls, settings)
If present, and ``from_crawler`` is not defined, this class method is called
to create a request fingerprinter instance from a
:class:`~scrapy.settings.Settings` object. It must return a new instance of
the request fingerprinter.
.. currentmodule:: scrapy.http
The :meth:`fingerprint` method of the default request fingerprinter,
:class:`scrapy.utils.request.RequestFingerprinter`, uses
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
For example, to take the value of a request header named ``X-ID`` into
account::
# my_project/settings.py
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
# my_project/utils.py
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
def fingerprint(self, request):
return fingerprint(request, include_headers=['X-ID'])
You can also write your own fingerprinting logic from scratch.
However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
- Caching saves CPU by ensuring that fingerprints are calculated only once
per request, and not once per Scrapy component that needs the fingerprint
of a request.
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
request objects do not stay in memory forever just because you have
references to them in your cache dictionary.
For example, to take into account only the URL of a request, without any prior
URL canonicalization or taking the request method or body into account::
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.url))
self.cache[request] = fp.digest()
return self.cache[request]
If you need to be able to override the request fingerprinting for arbitrary
requests from your spider callbacks, you may implement a request fingerprinter
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
when available, and then falls back to
:func:`scrapy.utils.request.fingerprint`. For example::
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
def fingerprint(self, request):
if 'fingerprint' in request.meta:
return request.meta['fingerprint']
return fingerprint(request)
If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
without using the deprecated ``'2.6'`` value of the
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
request fingerprinter::
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
from w3lib.url import canonicalize_url
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b'')
self.cache[request] = fp.digest()
return self.cache[request]
.. _request-fingerprint-restrictions:
Request fingerprint restrictions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Scrapy components that use request fingerprints may impose additional
restrictions on the format of the fingerprints that your :ref:`request
fingerprinter <custom-request-fingerprinter>` generates.
The following built-in Scrapy components have such restrictions:
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
value of :setting:`HTTPCACHE_STORAGE`)
Request fingerprints must be at least 1 byte long.
Path and filename length limits of the file system of
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
the following directory structure is created:
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
- first byte of a request fingerprint as hexadecimal
- fingerprint as hexadecimal
- filenames up to 16 characters long
For example, if a request fingerprint is made of 20 bytes (default),
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
and the name of your spider is ``'my_spider'`` your file system must
support a file path like::
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
The underlying DBM implementation must support keys as long as twice
the number of bytes of a request fingerprint, plus 5. For example,
if a request fingerprint is made of 20 bytes (default),
45-character-long keys must be supported.
.. _topics-request-meta:
Request.meta special keys
@ -351,26 +647,26 @@ are some special keys recognized by Scrapy and its built-in extensions.
Those are:
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`bindaddress`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`download_latency`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_timeout`
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`max_retry_times`
* :reqmeta:`proxy`
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`download_timeout`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_latency`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`proxy`
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* :reqmeta:`referrer_policy`
* :reqmeta:`max_retry_times`
.. reqmeta:: bindaddress
@ -420,9 +716,9 @@ The meta key is used set retry times per request. When initialized, the
Stopping the download of a Response
===================================
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
:class:`~scrapy.signals.bytes_received` signal handler will stop the
download of a given response. See the following example::
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signals will stop the download of a given response. See the following example::
import scrapy
@ -476,7 +772,9 @@ fields with form data from :class:`Response` objects.
.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
.. class:: FormRequest(url, [formdata, ...])
.. class:: scrapy.http.request.form.FormRequest
.. class:: scrapy.http.FormRequest
.. class:: scrapy.FormRequest(url, [formdata, ...])
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
@ -630,6 +928,8 @@ dealing with JSON requests.
data into JSON format.
:type dumps_kwargs: dict
.. autoattribute:: JsonRequest.attributes
JsonRequest usage example
-------------------------
@ -647,9 +947,6 @@ Response objects
.. autoclass:: Response
A :class:`Response` object represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
:param url: the URL of this response
:type url: str
@ -673,7 +970,7 @@ Response objects
:param request: the initial value of the :attr:`Response.request` attribute.
This represents the :class:`Request` that generated this response.
:type request: scrapy.http.Request
:type request: scrapy.Request
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
@ -681,9 +978,19 @@ Response objects
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
:param protocol: The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1", "h2"
:type protocol: :class:`str`
.. versionadded:: 2.0.0
The ``certificate`` parameter.
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. versionadded:: 2.5.0
The ``protocol`` parameter.
.. attribute:: Response.url
A string containing the URL of the response.
@ -768,6 +1075,8 @@ Response objects
.. attribute:: Response.certificate
.. versionadded:: 2.0.0
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
@ -783,6 +1092,19 @@ Response objects
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. attribute:: Response.protocol
.. versionadded:: 2.5.0
The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1"
This attribute is currently only populated by the HTTP download
handlers, i.e. for ``http(s)`` responses. For other handlers,
:attr:`protocol` is always ``None``.
.. autoattribute:: Response.attributes
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@ -876,9 +1198,11 @@ TextResponse objects
.. attribute:: TextResponse.selector
A :class:`~scrapy.selector.Selector` instance using the response as
A :class:`~scrapy.Selector` instance using the response as
target. The selector is lazily instantiated on first access.
.. autoattribute:: TextResponse.attributes
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
@ -903,6 +1227,14 @@ TextResponse objects
Returns a Python object from deserialized JSON document.
The result is cached after the first call.
.. method:: TextResponse.urljoin(url)
Constructs an absolute url by combining the Response's base url with
a possible relative url. The base url shall be extracted from the
``<base>`` tag, or just the Response's :attr:`url` if there is no such
tag.
HtmlResponse objects
--------------------

34
docs/topics/scheduler.rst Normal file
View File

@ -0,0 +1,34 @@
.. _topics-scheduler:
=========
Scheduler
=========
.. module:: scrapy.core.scheduler
The scheduler component receives requests from the :ref:`engine <component-engine>`
and stores them into persistent and/or non-persistent data structures.
It also gets those requests and feeds them back to the engine when it
asks for a next request to be downloaded.
Overriding the default scheduler
================================
You can use your own custom scheduler class by supplying its full
Python path in the :setting:`SCHEDULER` setting.
Minimal scheduler interface
===========================
.. autoclass:: BaseScheduler
:members:
Default Scrapy scheduler
========================
.. autoclass:: Scheduler
:members:
:special-members: __len__

26
docs/topics/selectors.rst
View File

@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is
to extract data from the HTML source. There are several libraries available to
achieve this, such as:
* `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
- `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
library.)
- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python
standard library.)
Scrapy comes with its own mechanism for extracting data. They're called
selectors because they "select" certain parts of the HTML document specified
@ -48,7 +48,7 @@ Constructing selectors
.. highlight:: python
Response objects expose a :class:`~scrapy.selector.Selector` instance
Response objects expose a :class:`~scrapy.Selector` instance
on ``.selector`` attribute:
>>> response.selector.xpath('//span/text()').get()
@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
>>> response.css('span::text').get()
'good'
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
Scrapy selectors are instances of :class:`~scrapy.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
markup as a string (in ``text`` argument).
@ -175,7 +175,7 @@ of ``None``:
'not-found'
Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
using ``.attrib`` property of a :class:`~scrapy.Selector`:
>>> [img.attrib['src'] for img in response.css('img')]
['image1_thumb.jpg',
@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page):
Using selectors with regular expressions
----------------------------------------
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
:class:`~scrapy.Selector` also has a ``.re()`` method for extracting
data using regular expressions. However, unlike using ``.xpath()`` or
``.css()`` methods, ``.re()`` returns a list of strings. So you
can't construct nested ``.re()`` calls.
@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet,
you may want to take a look first at this `XPath tutorial`_.
.. note::
Some of the tips are based on `this post from ScrapingHub's blog`_.
Some of the tips are based on `this post from Zyte's blog`_.
.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/
.. _topics-selectors-relative-xpaths:

180
docs/topics/settings.rst
View File

@ -67,7 +67,7 @@ Example::
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
own settings that will take precedence and override the project ones. They can
do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute::
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
class MySpider(scrapy.Spider):
name = 'myspider'
@ -98,11 +98,15 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Compatibility with pickle
=========================
Setting values must be :ref:`picklable <pickle-picklable>`.
Import paths and classes
========================
.. versionadded:: VERSION
.. versionadded:: 2.4.0
When a setting references a callable object to be imported by Scrapy, such as a
class or a function, there are two different ways you can specify that object:
@ -142,7 +146,7 @@ In a spider, the settings are available through ``self.settings``::
The ``settings`` attribute is set in the base Spider class after the spider
is initialized. If you want to use the settings before the initialization
(e.g., in your spider's ``__init__()`` method), you'll need to override the
:meth:`~scrapy.spiders.Spider.from_crawler` method.
:meth:`~scrapy.Spider.from_crawler` method.
Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
@ -204,6 +208,19 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
.. setting:: AWS_SESSION_TOKEN
AWS_SESSION_TOKEN
-----------------
Default: ``None``
The AWS security token used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
`temporary security credentials`_.
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
.. setting:: AWS_ENDPOINT_URL
AWS_ENDPOINT_URL
@ -249,19 +266,25 @@ ASYNCIO_EVENT_LOOP
Default: ``None``
Import path of a given asyncio event loop class.
Import path of a given ``asyncio`` event loop class.
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
asyncio event loop to be used with it. Set the setting to the import path of the
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
asyncio event loop to be used with it. Set the setting to the import path of the
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
event loop will be used.
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
class to be used.
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
class to be used.
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
.. caution:: Please be aware that, when using a non-default event loop
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
:func:`asyncio.set_event_loop`, which will set the specified event loop
as the current loop for the current OS thread.
.. setting:: BOT_NAME
BOT_NAME
@ -332,7 +355,7 @@ is non-zero, download delay is enforced per IP, not per domain.
DEFAULT_ITEM_CLASS
------------------
Default: ``'scrapy.item.Item'``
Default: ``'scrapy.Item'``
The default class that will be used for instantiating items in the :ref:`the
Scrapy shell <topics-shell>`.
@ -352,6 +375,11 @@ Default::
The default headers used for Scrapy HTTP Requests. They're populated in the
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
.. setting:: DEPTH_LIMIT
DEPTH_LIMIT
@ -373,8 +401,8 @@ Default: ``0``
Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
a :class:`~scrapy.http.Request` based on its depth.
An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
a :class:`~scrapy.Request` based on its depth.
The priority of a request is adjusted as follows::
@ -536,7 +564,6 @@ This setting must be one of these string values:
set this if you want the behavior of Scrapy<1.1
- ``'TLSv1.1'``: forces TLS version 1.1
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -646,6 +673,7 @@ DOWNLOAD_HANDLERS_BASE
Default::
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@ -666,6 +694,45 @@ handler (without replacement), place this in your ``settings.py``::
'ftp': None,
}
.. _http2:
The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
}
.. warning::
HTTP/2 support in Scrapy is experimental, and not yet recommended for
production environments. Future Scrapy versions may introduce related
changes without a deprecation period or warning.
.. note::
Known limitations of the current HTTP/2 implementation of Scrapy include:
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).
- No setting to specify a maximum `frame size`_ larger than the default
value, 16384. Connections to servers that send a larger frame will
fail.
- No support for `server pushes`_, which are ignored.
- No support for the :signal:`bytes_received` and
:signal:`headers_received` signals.
.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@ -743,6 +810,15 @@ Optionally, this can be set per-request basis by using the
If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
.. warning::
This setting is ignored by the
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
error, the corresponding HTTP/2 connection may be corrupted, affecting other
requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
failure is always raised for every request that was using that connection.
.. setting:: DUPEFILTER_CLASS
DUPEFILTER_CLASS
@ -752,18 +828,14 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
The class used to detect and filter duplicate requests.
The default (``RFPDupeFilter``) filters based on request fingerprint using
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
override its ``request_fingerprint`` method. This method should accept
scrapy :class:`~scrapy.http.Request` object and return its fingerprint
(a string).
The default (``RFPDupeFilter``) filters based on the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
Be very careful about this however, because you can get into crawling loops.
It's usually a better idea to set the ``dont_filter`` parameter to
``True`` on the specific :class:`~scrapy.http.Request` that should not be
``True`` on the specific :class:`~scrapy.Request` that should not be
filtered.
.. setting:: DUPEFILTER_DEBUG
@ -916,6 +988,16 @@ Default: ``{}``
A dict containing the pipelines enabled by default in Scrapy. You should never
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
.. setting:: JOBDIR
JOBDIR
------
Default: ``''``
A string indicating the directory for storing the state of a crawl when
:ref:`pausing and resuming crawls <topics-jobs>`.
.. setting:: LOG_ENABLED
LOG_ENABLED
@ -943,6 +1025,16 @@ Default: ``None``
File name to use for logging output. If ``None``, standard error will be used.
.. setting:: LOG_FILE_APPEND
LOG_FILE_APPEND
---------------
Default: ``True``
If ``False``, the log file specified with :setting:`LOG_FILE` will be
overwritten (discarding the output from previous runs, if any).
.. setting:: LOG_FORMAT
LOG_FORMAT
@ -1177,20 +1269,6 @@ Adjust redirect request priority relative to original request:
- **a positive priority adjust (default) means higher priority.**
- a negative priority adjust means lower priority.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. setting:: ROBOTSTXT_OBEY
ROBOTSTXT_OBEY
@ -1238,7 +1316,8 @@ SCHEDULER
Default: ``'scrapy.core.scheduler.Scheduler'``
The scheduler to use for crawling.
The scheduler class to be used for crawling.
See the :ref:`topics-scheduler` topic for details.
.. setting:: SCHEDULER_DEBUG
@ -1496,7 +1575,7 @@ If a reactor is already installed,
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
:exc:`Exception` if the installed reactor does not match the
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
:setting:`TWISTED_REACTOR` setting; therefore, having top-level
:mod:`~twisted.internet.reactor` imports in project files and imported
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
@ -1517,7 +1596,7 @@ In order to use the reactor installed by Scrapy::
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
urls = ['https://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
@ -1545,7 +1624,7 @@ which raises :exc:`Exception`, becomes::
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
urls = ['https://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
@ -1558,11 +1637,16 @@ which raises :exc:`Exception`, becomes::
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will not attempt to install any specific reactor, and the
default reactor defined by Twisted for the current platform will be used. This
means that Scrapy will use the existing reactor if one is already installed, or
install the default reactor defined by Twisted for the current platform. This
is to maintain backward compatibility and avoid possible problems caused by
using a non-default reactor.
.. versionchanged:: 2.7
The :command:`startproject` command now sets this setting to
``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
``settings.py`` file.
For additional information, see :doc:`core/howto/choosing-reactor`.
@ -1575,8 +1659,19 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
The maximum URL length to allow for crawled URLs.
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
:setting:`DEPTH_LIMIT`.
Use ``0`` to allow URLs of any length.
The default value is copied from the `Microsoft Internet Explorer maximum URL
length`_, even though this setting exists for different reasons.
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
.. setting:: USER_AGENT
@ -1588,7 +1683,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
The default User-Agent to use when crawling, unless overridden. This user agent is
also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
there is no overridding User-Agent header specified for the request.
there is no overriding User-Agent header specified for the request.
Settings documented elsewhere:
@ -1599,7 +1694,6 @@ case to see how to enable and use them.
.. settingslist::
.. _Amazon web services: https://aws.amazon.com/
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search

47
docs/topics/shell.rst
View File

@ -95,20 +95,21 @@ convenience.
Available Shortcuts
-------------------
* ``shelp()`` - print a help with the list of available objects and shortcuts
- ``shelp()`` - print a help with the list of available objects and
shortcuts
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
URL and update all related objects accordingly. You can optionaly ask for
HTTP 3xx redirections to not be followed by passing ``redirect=False``
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
and update all related objects accordingly. You can optionally ask for HTTP
3xx redirections to not be followed by passing ``redirect=False``
* ``fetch(request)`` - fetch a new response from the given request and
update all related objects accordingly.
- ``fetch(request)`` - fetch a new response from the given request and update
all related objects accordingly.
* ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
- ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
.. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
@ -117,26 +118,26 @@ Available Scrapy objects
The Scrapy shell automatically creates some convenient objects from the
downloaded page, like the :class:`~scrapy.http.Response` object and the
:class:`~scrapy.selector.Selector` objects (for both HTML and XML
:class:`~scrapy.Selector` objects (for both HTML and XML
content).
Those objects are:
* ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
* ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.spiders.Spider` object if there is no spider found for
the current URL
- ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.Spider` object if there is no spider found for the
current URL
* ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
page. You can modify this request using :meth:`~scrapy.http.Request.replace`
or fetch a new request (without leaving the shell) using the ``fetch``
shortcut.
- ``request`` - a :class:`~scrapy.Request` object of the last fetched
page. You can modify this request using
:meth:`~scrapy.Request.replace` or fetch a new request (without
leaving the shell) using the ``fetch`` shortcut.
* ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
- ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
* ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
- ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
Example of shell session
========================

111
docs/topics/signals.rst
View File

@ -51,16 +51,16 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
or :term:`awaitable objects <awaitable>` from their handlers, allowing
you to run asynchronous code that does not block Scrapy. If a signal
handler returns one of these objects, Scrapy waits for that asynchronous
operation to finish.
Let's take an example::
Let's take an example using :ref:`coroutines <topics-coroutines>`::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['http://quotes.toscrape.com/page/1/']
start_urls = ['https://quotes.toscrape.com/page/1/']
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
@ -68,17 +68,15 @@ Let's take an example::
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
async def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
response = await treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
# The next item will be scraped only after
# deferred (d) is fired
return d
return response
def parse(self, response):
for quote in response.css('div.quote'):
@ -89,7 +87,7 @@ Let's take an example::
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
.. _topics-signals-ref:
@ -155,7 +153,7 @@ item_scraped
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param response: the response from where the item was scraped
:type response: :class:`~scrapy.http.Response` object
@ -175,7 +173,7 @@ item_dropped
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param response: the response from where the item was dropped
:type response: :class:`~scrapy.http.Response` object
@ -203,7 +201,7 @@ item_error
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
@ -223,7 +221,7 @@ spider_closed
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param reason: a string which describes the reason why the spider was closed. If
it was closed because the spider has completed scraping, the reason
@ -247,7 +245,7 @@ spider_opened
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
spider_idle
~~~~~~~~~~~
@ -268,10 +266,17 @@ spider_idle
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
exception to provide a custom spider closing reason. An
idle handler is the perfect place to put some code that assesses
the final spider results and update the final closing reason
accordingly (e.g. setting it to 'too_few_results' instead of
'finished').
This signal does not support returning deferreds from its handlers.
:param spider: the spider which has gone idle
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. note:: Scheduling some requests in your :signal:`spider_idle` handler does
**not** guarantee that it can prevent the spider from being closed,
@ -296,7 +301,7 @@ spider_error
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
Request signals
---------------
@ -307,16 +312,16 @@ request_scheduled
.. signal:: request_scheduled
.. function:: request_scheduled(request, spider)
Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
Sent when the engine schedules a :class:`~scrapy.Request`, to be
downloaded later.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
request_dropped
~~~~~~~~~~~~~~~
@ -324,16 +329,16 @@ request_dropped
.. signal:: request_dropped
.. function:: request_dropped(request, spider)
Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
Sent when a :class:`~scrapy.Request`, scheduled by the engine to be
downloaded later, is rejected by the scheduler.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
request_reached_downloader
~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -341,15 +346,15 @@ request_reached_downloader
.. signal:: request_reached_downloader
.. function:: request_reached_downloader(request, spider)
Sent when a :class:`~scrapy.http.Request` reached downloader.
Sent when a :class:`~scrapy.Request` reached downloader.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached downloader
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
request_left_downloader
~~~~~~~~~~~~~~~~~~~~~~~
@ -359,16 +364,16 @@ request_left_downloader
.. versionadded:: 2.0
Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of
failure.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the downloader
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
bytes_received
~~~~~~~~~~~~~~
@ -384,22 +389,52 @@ bytes_received
a possible scenario for a 25 kb response would be two signals fired
with 10 kb of data, and a final one with 5 kb of data.
Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type data: :class:`bytes` object
:param request: the request that generated the download
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. note:: Handlers of this signal can stop the download of a response while it
headers_received
~~~~~~~~~~~~~~~~
.. versionadded:: 2.5
.. signal:: headers_received
.. function:: headers_received(headers, body_length, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
available for a given request, before downloading any additional content.
Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
This signal does not support returning deferreds from its handlers.
:param headers: the headers received by the download handler
:type headers: :class:`scrapy.http.headers.Headers` object
:param body_length: expected size of the response body, in bytes
:type body_length: `int`
:param request: the request that generated the download
:type request: :class:`~scrapy.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
Response signals
----------------
@ -418,10 +453,10 @@ response_received
:type response: :class:`~scrapy.http.Response` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. note:: The ``request`` argument might not contain the original request that
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
@ -442,7 +477,7 @@ response_downloaded
:type response: :class:`~scrapy.http.Response` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object

54
docs/topics/spider-middleware.rst
View File

@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output(response, result, spider)
@ -102,19 +102,39 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.http.Request` objects and :ref:`item object
:class:`~scrapy.Request` objects and :ref:`item objects
<topics-items>`.
.. versionchanged:: 2.7
This method may be defined as an :term:`asynchronous generator`, in
which case ``result`` is an :term:`asynchronous iterable`.
Consider defining this method as an :term:`asynchronous generator`,
which will be a requirement in a future version of Scrapy. However, if
you plan on sharing your spider middleware with other people, consider
either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
as a minimum requirement of your spider middleware, or :ref:`making
your spider middleware universal <universal-spider-middleware>` so that
it works with Scrapy versions earlier than Scrapy 2.7.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.http.Request` objects and
:ref:`item object <topics-items>`
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item objects <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output_async(response, result, spider)
.. versionadded:: 2.7
If defined, this method must be an :term:`asynchronous generator`,
which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`.
.. method:: process_spider_exception(response, exception, spider)
@ -122,8 +142,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
<topics-items>`.
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
objects.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_spider_exception` in the following
@ -142,7 +162,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type exception: :exc:`Exception` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_start_requests(start_requests, spider)
@ -152,7 +172,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
items).
It receives an iterable (in the ``start_requests`` parameter) and must
return another iterable of :class:`~scrapy.http.Request` objects.
return another iterable of :class:`~scrapy.Request` objects.
.. note:: When implementing this method in your spider middleware, you
should always return an iterable (that follows the input one) and
@ -164,10 +184,10 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
(like a time limit or item/page count).
:param start_requests: the start requests
:type start_requests: an iterable of :class:`~scrapy.http.Request`
:type start_requests: an iterable of :class:`~scrapy.Request`
:param spider: the spider to whom the start requests belong
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
@ -251,9 +271,10 @@ this::
.. reqmeta:: handle_httpstatus_all
The ``handle_httpstatus_list`` key of :attr:`Request.meta
<scrapy.http.Request.meta>` can also be used to specify which response codes to
<scrapy.Request.meta>` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
to ``True`` if you want to allow any response code for a request.
to ``True`` if you want to allow any response code for a request, and ``False`` to
disable the effects of the ``handle_httpstatus_all`` key.
Keep in mind, however, that it's usually a bad idea to handle non-200
responses, unless you really know what you're doing.
@ -294,7 +315,7 @@ OffsiteMiddleware
Filters out Requests for URLs outside the domains covered by the spider.
This middleware filters out every request whose host names aren't in the
spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
All subdomains of any domain in the list are also allowed.
E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
but not ``www2.example.com`` nor ``example.com``.
@ -312,10 +333,10 @@ OffsiteMiddleware
will be printed (but only for the first request filtered).
If the spider doesn't define an
:attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the
:attr:`~scrapy.Spider.allowed_domains` attribute, or the
attribute is empty, the offsite middleware will allow all requests.
If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute
If the request has the :attr:`~scrapy.Request.dont_filter` attribute
set, the offsite middleware will allow the request even if its domain is not
listed in allowed domains.
@ -439,4 +460,3 @@ UrlLengthMiddleware
settings (see the settings documentation for more info):
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.

82
docs/topics/spiders.rst
View File

@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this:
those requests.
The first requests to perform are obtained by calling the
:meth:`~scrapy.spiders.Spider.start_requests` method which (by default)
generates :class:`~scrapy.http.Request` for the URLs specified in the
:attr:`~scrapy.spiders.Spider.start_urls` and the
:attr:`~scrapy.spiders.Spider.parse` method as callback function for the
:meth:`~scrapy.Spider.start_requests` method which (by default)
generates :class:`~scrapy.Request` for the URLs specified in the
:attr:`~scrapy.Spider.start_urls` and the
:attr:`~scrapy.Spider.parse` method as callback function for the
Requests.
2. In the callback function, you parse the response (web page) and return
:ref:`item objects <topics-items>`,
:class:`~scrapy.http.Request` objects, or an iterable of these objects.
:class:`~scrapy.Request` objects, or an iterable of these objects.
Those Requests will also contain a callback (maybe
the same) and will then be downloaded by Scrapy and then their
response handled by the specified callback.
@ -42,15 +42,13 @@ Even though this cycle applies (more or less) to any kind of spider, there are
different kinds of default spiders bundled into Scrapy for different purposes.
We will talk about those types here.
.. module:: scrapy.spiders
:synopsis: Spiders base class, spider manager and spider middleware
.. _topics-spiders-ref:
scrapy.Spider
=============
.. class:: Spider()
.. class:: scrapy.spiders.Spider
.. class:: scrapy.Spider()
This is the simplest spider, and the one from which every other spider
must inherit (including spiders that come bundled with Scrapy, as well as spiders
@ -86,7 +84,7 @@ scrapy.Spider
A list of URLs where the spider will begin to crawl from, when no
particular URLs are specified. So, the first pages downloaded will be those
listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data
contained in the start URLs.
.. attribute:: custom_settings
@ -121,6 +119,11 @@ scrapy.Spider
send log messages through it as described on
:ref:`topics-logging-from-spiders`.
.. attribute:: state
A dict you can use to persist some spider state between batches.
See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
.. method:: from_crawler(crawler, *args, **kwargs)
This is the class method used by Scrapy to create your spiders.
@ -178,9 +181,10 @@ scrapy.Spider
scraped data and/or more URLs to follow. Other Requests callbacks have
the same requirements as the :class:`Spider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
<topics-items>`.
This method, as well as any other Request callback, must return a
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
<topics-items>`, or ``None``.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`
@ -234,7 +238,7 @@ Return multiple Requests and items from a single callback::
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :class:`~scrapy.item.Item` objects::
to give data more structure you can use :class:`~scrapy.Item` objects::
import scrapy
from myproject.items import MyItem
@ -294,6 +298,14 @@ The above example can also be written as follows::
def start_requests(self):
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
If you were to set the ``start_urls`` attribute from the command line,
@ -358,14 +370,14 @@ CrawlSpider
described below. If multiple rules match the same link, the first one
will be used, according to the order they're defined in this attribute.
This spider also exposes an overrideable method:
This spider also exposes an overridable method:
.. method:: parse_start_url(response, **kwargs)
This method is called for each response produced for the URLs in
the spider's ``start_urls`` attribute. It allows to parse
the initial responses and must return either an
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
:ref:`item object <topics-items>`, a :class:`~scrapy.Request`
object, or an iterable containing any of them.
Crawling rules
@ -375,7 +387,7 @@ Crawling rules
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
defines how links will be extracted from each crawled page. Each produced link will
be used to generate a :class:`~scrapy.http.Request` object, which will contain the
be used to generate a :class:`~scrapy.Request` object, which will contain the
link's text in its ``meta`` dictionary (under the ``link_text`` key).
If omitted, a default link extractor created with no arguments will be used,
resulting in all links being extracted.
@ -384,9 +396,9 @@ Crawling rules
object with that name will be used) to be called for each link extracted with
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
as its first argument and must return either a single instance or an iterable of
:ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
:ref:`item objects <topics-items>` and/or :class:`~scrapy.Request` objects
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
object will contain the text of the link that produced the :class:`~scrapy.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
@ -403,7 +415,7 @@ Crawling rules
``process_request`` is a callable (or a string, in which case a method from
the spider object with that name will be used) which will be called for every
:class:`~scrapy.http.Request` extracted by this rule. This callable should
:class:`~scrapy.Request` extracted by this rule. This callable should
take said request as first argument and the :class:`~scrapy.http.Response`
from which the request originated as second argument. It must return a
``Request`` object or ``None`` (to filter out the request).
@ -414,10 +426,9 @@ Crawling rules
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
unexpected behaviour can occur otherwise.
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
unexpected behaviour can occur otherwise.
.. versionadded:: 2.0
The *errback* parameter.
@ -463,7 +474,7 @@ Let's now take a look at an example CrawlSpider with rules::
This spider would start crawling example.com's home page, collecting category
links, and item links, parsing the latter with the ``parse_item`` method. For
each item response, some data will be extracted from the HTML using XPath, and
an :class:`~scrapy.item.Item` will be filled with it.
an :class:`~scrapy.Item` will be filled with it.
XMLFeedSpider
-------------
@ -486,11 +497,11 @@ XMLFeedSpider
- ``'iternodes'`` - a fast iterator based on regular expressions
- ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
Keep in mind this uses DOM parsing and must load all DOM in memory
which could be a problem for big feeds
- ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
- ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`.
Keep in mind this uses DOM parsing and must load all DOM in memory
which could be a problem for big feeds
@ -508,7 +519,7 @@ XMLFeedSpider
available in that document that will be processed with this spider. The
``prefix`` and ``uri`` will be used to automatically register
namespaces using the
:meth:`~scrapy.selector.Selector.register_namespace` method.
:meth:`~scrapy.Selector.register_namespace` method.
You can then specify nodes with namespaces in the :attr:`itertag`
attribute.
@ -521,7 +532,7 @@ XMLFeedSpider
itertag = 'n:url'
# ...
Apart from these new attributes, this spider has the following overrideable
Apart from these new attributes, this spider has the following overridable
methods too:
.. method:: adapt_response(response)
@ -535,10 +546,10 @@ XMLFeedSpider
This method is called for the nodes matching the provided tag name
(``itertag``). Receives the response and an
:class:`~scrapy.selector.Selector` for each node. Overriding this
:class:`~scrapy.Selector` for each node. Overriding this
method is mandatory. Otherwise, you spider won't work. This method
must return an :ref:`item object <topics-items>`, a
:class:`~scrapy.http.Request` object, or an iterable containing any of
:class:`~scrapy.Request` object, or an iterable containing any of
them.
.. method:: process_results(response, results)
@ -549,10 +560,9 @@ XMLFeedSpider
item IDs. It receives a list of results and the response which originated
those results. It must return a list of results (items or requests).
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
unexpected behaviour can occur otherwise.
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
unexpected behaviour can occur otherwise.
XMLFeedSpider example
@ -581,7 +591,7 @@ These spiders are pretty easy to use, let's have a look at one example::
Basically what we did up there was to create a spider that downloads a feed from
the given ``start_urls``, and then iterates through each of its ``item`` tags,
prints them out, and stores some random data in an :class:`~scrapy.item.Item`.
prints them out, and stores some random data in an :class:`~scrapy.Item`.
CSVFeedSpider
-------------

3
docs/topics/telnetconsole.rst
View File

@ -110,11 +110,10 @@ using the telnet console::
Execution engine status
time()-engine.start_time : 8.62972998619
engine.has_capacity() : False
len(engine.downloader.active) : 16
engine.scraper.is_idle() : False
engine.spider.name : followall
engine.spider_is_idle(engine.spider) : False
engine.spider_is_idle() : False
engine.slot.closing : False
len(engine.slot.inprogress) : 16
len(engine.slot.scheduler.dqs or []) : 0

11
docs/topics/webservice.rst
View File

@ -1,11 +0,0 @@
.. _topics-webservice:
===========
Web Service
===========
webservice has been moved into a separate project.
It is hosted at:
https://github.com/scrapy-plugins/scrapy-jsonrpc

2
docs/versioning.rst
View File

@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
large changes.
* *B* is the release number. This will include many changes including features
and things that possibly break backward compatibility, although we strive to
keep theses cases at a minimum.
keep these cases at a minimum.
* *C* is the bugfix release number.
Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,

2
extras/qpsclient.py
View File

@ -1,5 +1,5 @@
"""
A spider that generate light requests to meassure QPS troughput
A spider that generate light requests to meassure QPS throughput
usage:

14
pylintrc
View File

@ -6,13 +6,12 @@ jobs=1 # >1 hides results
disable=abstract-method,
anomalous-backslash-in-string,
arguments-differ,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-continuation,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bad-whitespace,
bare-except,
blacklisted-name,
broad-except,
@ -21,9 +20,12 @@ disable=abstract-method,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-dict-items,
consider-using-from-import,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
@ -45,10 +47,10 @@ disable=abstract-method,
keyword-arg-before-vararg,
line-too-long,
logging-format-interpolation,
logging-fstring-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
misplaced-comparison-constant,
missing-docstring,
missing-final-newline,
multiple-imports,
@ -56,12 +58,10 @@ disable=abstract-method,
no-else-continue,
no-else-raise,
no-else-return,
no-init,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-self-use,
no-value-for-parameter,
not-an-iterable,
not-callable,
@ -98,14 +98,18 @@ disable=abstract-method,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-dunder-call,
unnecessary-lambda,
unnecessary-pass,
unreachable,
unspecified-encoding,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,

29
pytest.ini
View File

@ -1,4 +1,5 @@
[pytest]
xfail_strict = true
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
@ -17,27 +18,11 @@ addopts =
--ignore=docs/topics/stats.rst
--ignore=docs/topics/telnetconsole.rst
--ignore=docs/utils
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-max-line-length = 119
flake8-ignore =
W503
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/utils/http.py F403
scrapy/utils/markup.py F403
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
ignore:typing.re is deprecated
ignore:typing.io is deprecated

2
scrapy/VERSION
View File

@ -1 +1 @@
2.3.0
2.7.1

6
scrapy/__init__.py
View File

@ -22,14 +22,14 @@ __all__ = [
# Scrapy and Twisted versions
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 6):
print("Scrapy %s requires Python 3.6+" % __version__)
if sys.version_info < (3, 7):
print(f"Scrapy {__version__} requires Python 3.7+")
sys.exit(1)

23
scrapy/cmdline.py
View File

@ -1,19 +1,28 @@
import sys
import os
import optparse
import argparse
import cProfile
import inspect
import pkg_resources
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == '-:':
return None
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# scrapy.utils.spider.iter_spider_classes
@ -123,8 +132,6 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
sys.exit(0)
@ -133,12 +140,14 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
cmd.add_options(parser)
opts, args = parser.parse_args(args=argv[1:])
opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
cmd.crawler_process = CrawlerProcess(settings)

85
scrapy/commands/__init__.py
View File

@ -2,7 +2,9 @@
Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
import argparse
from typing import Any, Dict
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@ -15,7 +17,7 @@ class ScrapyCommand:
crawler_process = None
# default settings to be used for this command instead of global defaults
default_settings = {}
default_settings: Dict[str, Any] = {}
exitcode = 0
@ -41,14 +43,14 @@ class ScrapyCommand:
def long_desc(self):
"""A long description of the command. Return short description when not
available. It cannot contain newlines, since contents will be formatted
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
"""
return self.short_desc()
def help(self):
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines, since no post-formatting will
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
return self.long_desc()
@ -57,22 +59,20 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)
group = parser.add_argument_group(title='Global Options')
group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_argument("--nolog", action="store_true",
help="disable logging completely")
group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
@ -112,14 +112,16 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -135,3 +137,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
return super()._join_parts(parts)
def format_part_strings(self, part_strings):
"""
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

2
scrapy/commands/bench.py
View File

@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

8
scrapy/commands/check.py
View File

@ -49,10 +49,10 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
def run(self, args, opts):
# load contracts

2
scrapy/commands/crawl.py
View File

@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)

10
scrapy/commands/fetch.py
View File

@ -26,11 +26,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():

34
scrapy/commands/genspider.py
View File

@ -4,6 +4,7 @@ import string
from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext
from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
@ -22,6 +23,14 @@ def sanitize_module_name(module_name):
return module_name
def extract_domain(url):
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == '' and o.netloc == '':
o = urlparse("//" + url.lstrip("/"))
return o.netloc
class Command(ScrapyCommand):
requires_project = False
@ -35,16 +44,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:
@ -59,7 +68,8 @@ class Command(ScrapyCommand):
if len(args) != 2:
raise UsageError()
name, domain = args[0:2]
name, url = args[0:2]
domain = extract_domain(url)
module = sanitize_module_name(name)
if self.settings.get('BOT_NAME') == module:
@ -98,7 +108,7 @@ class Command(ScrapyCommand):
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n {spiders_module.__name__}.{module}")
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, f'{template}.tmpl')

108
scrapy/commands/parse.py
View File

@ -1,15 +1,19 @@
import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from twisted.internet.defer import maybeDeferred
from scrapy.commands import BaseRunSpiderCommand
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)
@ -17,8 +21,8 @@ class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
items = {}
requests = {}
items: Dict[int, list] = {}
requests: Dict[int, list] = {}
first_response = None
@ -30,28 +34,28 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %(default)s]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@property
def max_level(self):
@ -108,16 +112,19 @@ class Command(BaseRunSpiderCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
items, requests = [], []
for x in iterate_spider_output(callback(response, **cb_kwargs)):
for x in spider_output:
if is_item(x):
items.append(x)
elif isinstance(x, Request):
requests.append(x)
return items, requests
return items, requests, opts, depth, spider, callback
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(self, spider, response):
if getattr(spider, 'rules', None):
@ -144,7 +151,8 @@ class Command(BaseRunSpiderCommand):
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
if self.spidercls:
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
self.crawler_process.crawl(self.spidercls, **opts.spargs)
@ -155,6 +163,25 @@ class Command(BaseRunSpiderCommand):
logger.error('No response downloaded for: %(url)s',
{'url': url})
def scraped_data(self, args):
items, requests, opts, depth, spider, callback = args
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, spider)
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
scraped_data += requests
return scraped_data
def prepare_request(self, spider, request, opts):
def callback(response, **cb_kwargs):
# memorize first request
@ -188,23 +215,10 @@ class Command(BaseRunSpiderCommand):
# parse items and requests
depth = response.meta['_depth']
items, requests = self.run_callback(response, cb, cb_kwargs)
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, spider)
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
scraped_data += requests
return scraped_data
d = self.run_callback(response, cb, cb_kwargs)
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
d.addCallback(self.scraped_data)
return d
# update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:

2
scrapy/commands/runspider.py
View File

@ -11,7 +11,7 @@ def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext != '.py':
if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path

20
scrapy/commands/settings.py
View File

@ -18,16 +18,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings

14
scrapy/commands/shell.py
View File

@ -33,12 +33,12 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_argument("--spider", dest="spider",
help="use this spider")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
@ -75,6 +75,6 @@ class Command(ScrapyCommand):
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
kwargs={'stop_after_crawl': False})
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
t.daemon = True
t.start()

9
scrapy/commands/startproject.py
View File

@ -1,7 +1,7 @@
import re
import os
import string
from importlib import import_module
from importlib.util import find_spec
from os.path import join, exists, abspath
from shutil import ignore_patterns, move, copy2, copystat
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@ -42,11 +42,8 @@ class Command(ScrapyCommand):
def _is_valid_name(self, project_name):
def _module_exists(module_name):
try:
import_module(module_name)
return True
except ImportError:
return False
spec = find_spec(module_name)
return spec is not None and spec.loader is not None
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print('Error: Project names must begin with a letter and contain'

4
scrapy/commands/version.py
View File

@ -16,8 +16,8 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
if opts.verbose:

3
scrapy/commands/view.py
View File

@ -1,3 +1,4 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
@ -12,7 +13,7 @@ class Command(fetch.Command):
def add_options(self, parser):
super().add_options(parser)
parser.remove_option("--headers")
parser.add_argument('--headers', help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)

127
scrapy/contracts/__init__.py
View File

@ -1,16 +1,77 @@
import sys
import re
import sys
from functools import wraps
from inspect import getmembers
from typing import Dict
from unittest import TestCase
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
from scrapy.utils.spider import iterate_spider_output
class Contract:
""" Abstract class for contracts """
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
class ContractsManager:
contracts = {}
contracts: Dict[str, Contract] = {}
def __init__(self, contracts):
for contract in contracts:
@ -107,66 +168,6 @@ class ContractsManager:
request.errback = eb_wrapper
class Contract:
""" Abstract class for contracts """
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
def _create_testcase(method, desc):
spider = method.__self__.name

59
scrapy/core/downloader/contextfactory.py
View File

@ -1,10 +1,15 @@
import warnings
from OpenSSL import SSL
from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
from scrapy.utils.misc import create_instance, load_object
@implementer(IPolicyForHTTPS)
@ -16,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
which allows TLS protocol negotiation
'A TLS/SSL connection established with [this method] may
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
@ -81,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
The default OpenSSL method is ``TLS_METHOD`` (also called
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
def creatorForNetloc(self, hostname, port):
def creatorForNetloc(self, hostname, port):
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@ -92,3 +97,51 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)
@implementer(IPolicyForHTTPS)
class AcceptableProtocolsContextFactory:
"""Context factory to used to override the acceptable protocols
to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN
negotiation.
"""
def __init__(self, context_factory, acceptable_protocols):
verifyObject(IPolicyForHTTPS, context_factory)
self._wrapped_context_factory = context_factory
self._acceptable_protocols = acceptable_protocols
def creatorForNetloc(self, hostname, port):
options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)
return options
def load_context_factory_from_settings(settings, crawler):
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
method=ssl_method,
)
except TypeError:
# use context factory defaults
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
)
msg = (
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
"a `method` argument (type OpenSSL.SSL method, e.g. "
"OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
"argument and/or a `tls_ciphers` argument. Please, upgrade your "
"context factory class to handle them or ignore them."
)
warnings.warn(msg)
return context_factory

6
scrapy/core/downloader/handlers/ftp.py
View File

@ -102,11 +102,11 @@ class FTPDownloadHandler:
def _build_response(self, result, request, protocol):
self.result = result
respcls = responsetypes.from_args(url=request.url)
protocol.close()
body = protocol.filename or protocol.body.read()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
body = to_bytes(protocol.filename or protocol.body.read())
respcls = responsetypes.from_args(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
def _failed(self, result, request):
message = result.getErrorMessage()

102
scrapy/core/downloader/handlers/http11.py
View File

@ -7,7 +7,7 @@ import warnings
from contextlib import suppress
from io import BytesIO
from time import time
from urllib.parse import urldefrag
from urllib.parse import urldefrag, urlunparse
from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.tls import openssl_methods
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
self._contextFactory = create_instance(
objcls=self._contextFactoryClass,
settings=settings,
crawler=crawler,
method=self._sslMethod,
)
except TypeError:
# use context factory defaults
self._contextFactory = create_instance(
objcls=self._contextFactoryClass,
settings=settings,
crawler=crawler,
)
msg = f"""
'{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \
argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \
`tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them."""
warnings.warn(msg)
self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
@ -121,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
with this endpoint comes from the pool and a CONNECT has already been issued
for it.
"""
_responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
_truncatedLength = 1000
_responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
_responseMatcher = re.compile(_responseAnswer.encode())
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
@ -167,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
else:
extra = rcvd_bytes[:32]
extra = rcvd_bytes[:self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')
@ -235,7 +213,7 @@ class TunnelingAgent(Agent):
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
key += self._proxyConf
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
@ -298,16 +276,19 @@ class ScrapyAgent:
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
_, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Crawlera, it doesn't require this mode anymore, "
"so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.",
ScrapyDeprecationWarning)
warnings.warn(
"Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Zyte Smart Proxy Manager, it doesn't require "
"this mode anymore, so you should update scrapy-crawlera "
"to scrapy-zyte-smartproxy and remove '?noconnect' "
"from the Zyte Smart Proxy Manager URL.",
ScrapyDeprecationWarning,
)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
@ -320,9 +301,13 @@ class ScrapyAgent:
pool=self._pool,
)
else:
proxyScheme = proxyScheme or b'http'
proxyHost = to_bytes(proxyHost, encoding='ascii')
proxyPort = to_bytes(str(proxyPort), encoding='ascii')
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxy, encoding='ascii'),
proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
@ -378,7 +363,37 @@ class ScrapyAgent:
request.meta['download_latency'] = time() - start_time
return result
@staticmethod
def _headers_from_twisted_response(response):
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b'Content-Length'] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
def _cb_bodyready(self, txresponse, request):
headers_received_result = self._crawler.signals.send_catch_log(
signal=signals.headers_received,
headers=self._headers_from_twisted_response(txresponse),
body_length=txresponse.length,
request=request,
spider=self._crawler.spider,
)
for handler, result in headers_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__})
txresponse._transport.stopProducing()
txresponse._transport.loseConnection()
return {
"txresponse": txresponse,
"body": b"",
"flags": ["download_stopped"],
"certificate": None,
"ip_address": None,
"failure": result if result.value.fail else None,
}
# deliverBody hangs for responses without body
if txresponse.length == 0:
return {
@ -401,7 +416,7 @@ class ScrapyAgent:
logger.warning(warning_msg, warning_args)
txresponse._transport._producer.loseConnection()
txresponse._transport.loseConnection()
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
@ -432,8 +447,13 @@ class ScrapyAgent:
return d
def _cb_bodydone(self, result, request, url):
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
headers = self._headers_from_twisted_response(result["txresponse"])
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
try:
version = result["txresponse"].version
protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
except (AttributeError, TypeError, IndexError):
protocol = None
response = respcls(
url=url,
status=int(result["txresponse"].code),
@ -442,6 +462,7 @@ class ScrapyAgent:
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
protocol=protocol,
)
if result.get("failure"):
result["failure"].value.response = response
@ -520,7 +541,8 @@ class _ResponseReader(protocol.Protocol):
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
self.transport._producer.loseConnection()
self.transport.stopProducing()
self.transport.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)

129
scrapy/core/downloader/handlers/http2.py Normal file
View File

@ -0,0 +1,129 @@
import warnings
from time import time
from typing import Optional, Type, TypeVar
from urllib.parse import urldefrag
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
class H2DownloadHandler:
def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
self._crawler = crawler
from twisted.internet import reactor
self._pool = H2ConnectionPool(reactor, settings)
self._context_factory = load_context_factory_from_settings(settings, crawler)
@classmethod
def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
agent = ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
crawler=self._crawler,
)
return agent.download_request(request, spider)
def close(self) -> None:
self._pool.close_connections()
class ScrapyH2Agent:
_Agent = H2Agent
_ProxyAgent = ScrapyProxyH2Agent
def __init__(
self, context_factory,
pool: H2ConnectionPool,
connect_timeout: int = 10,
bind_address: Optional[bytes] = None,
crawler: Optional[Crawler] = None,
) -> None:
self._context_factory = context_factory
self._connect_timeout = connect_timeout
self._bind_address = bind_address
self._pool = pool
self._crawler = crawler
def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
from twisted.internet import reactor
bind_address = request.meta.get('bindaddress') or self._bind_address
proxy = request.meta.get('proxy')
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = proxy_host.decode()
omit_connect_tunnel = b'noconnect' in proxy_params
if omit_connect_tunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is not "
"supported by the downloader handler. If you use Zyte "
"Smart Proxy Manager, it doesn't require this mode "
"anymore, so you should update scrapy-crawlera to "
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
"Zyte Smart Proxy Manager URL."
)
if scheme == b'https' and not omit_connect_tunnel:
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
)
return self._Agent(
reactor=reactor,
context_factory=self._context_factory,
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
)
def download_request(self, request: Request, spider: Spider) -> Deferred:
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connect_timeout
agent = self._get_agent(request, timeout)
start_time = time()
d = agent.request(request, spider)
d.addCallback(self._cb_latency, request, start_time)
timeout_cl = reactor.callLater(timeout, d.cancel)
d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
return d
@staticmethod
def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
request.meta['download_latency'] = time() - start_time
return response
@staticmethod
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
if timeout_cl.active():
timeout_cl.cancel()
return response
url = urldefrag(request.url)[0]
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")

72
scrapy/core/downloader/handlers/s3.py
View File

@ -1,46 +1,26 @@
from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
def _get_boto_connection():
from boto.s3.connection import S3Connection
class _v19_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, method, bucket, key, headers, *args, **kwargs):
return headers
class _v20_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, http_request, *args, **kwargs):
http_request.authorize(connection=self)
return http_request.headers
try:
import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
_S3Connection = _v20_S3Connection
return _S3Connection
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
if not aws_session_token:
aws_session_token = settings['AWS_SESSION_TOKEN']
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
@ -51,23 +31,15 @@ class S3DownloadHandler:
self.anon = kw.get('anon')
self._signer = None
if is_botocore():
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
else:
_S3Connection = _get_boto_connection()
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key, aws_session_token))
_http_handler = create_instance(
objcls=httpdownloadhandler,
@ -88,7 +60,7 @@ class S3DownloadHandler:
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
else:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
@ -98,14 +70,4 @@ class S3DownloadHandler:
self._signer.add_auth(awsrequest)
request = request.replace(
url=url, headers=awsrequest.headers.items())
else:
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body,
)
request = request.replace(url=url, headers=signed_headers)
return self._download_http(request, spider)

26
scrapy/core/downloader/middleware.py
View File

@ -3,8 +3,12 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
from twisted.internet import defer
from typing import Callable, Union, cast
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
@ -29,15 +33,15 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].appendleft(mw.process_exception)
def download(self, download_func, request, spider):
def download(self, download_func: Callable, request: Request, spider: Spider):
@defer.inlineCallbacks
def process_request(request):
def process_request(request: Request):
for method in self.methods['process_request']:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__self__.__class__.__name__}"
".process_request must return None, Response or "
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
@ -45,18 +49,18 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
def process_response(response):
def process_response(response: Union[Response, Request]):
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
return response
for method in self.methods['process_response']:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__self__.__class__.__name__}"
".process_response must return Response or Request, "
f"Middleware {method.__qualname__} must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
@ -64,14 +68,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
@defer.inlineCallbacks
def process_exception(failure):
def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__self__.__class__.__name__}"
".process_exception must return None, Response or "
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {type(response)}"
)
if response:

37
scrapy/core/downloader/tls.py
View File

@ -5,14 +5,12 @@ from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
from twisted.internet.ssl import AcceptableCiphers
from scrapy import twisted_version
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)
METHOD_SSLv3 = 'SSLv3'
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
METHOD_TLSv11 = 'TLSv1.1'
@ -21,20 +19,12 @@ METHOD_TLSv12 = 'TLSv1.2'
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
}
if twisted_version < (17, 0, 0):
from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
else:
def set_tlsext_host_name(connection, hostNameBytes):
connection.set_tlsext_host_name(hostNameBytes)
class ScrapyClientTLSOptions(ClientTLSOptions):
"""
SSL Client connection creator ignoring certificate verification errors
@ -52,21 +42,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL.SSL_CB_HANDSHAKE_START:
set_tlsext_host_name(connection, self._hostnameBytes)
connection.set_tlsext_host_name(self._hostnameBytes)
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
else:
logger.debug('SSL connection to %s using cipher %s',
self._hostnameASCII,
connection.get_cipher_name(),
)
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),
@ -80,14 +63,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
verifyHostname(connection, self._hostnameASCII)
except (CertificateError, VerificationError) as e:
logger.warning(
'Remote certificate is not valid for hostname "{}"; {}'.format(
self._hostnameASCII, e))
'Remote certificate is not valid for hostname "%s"; %s',
self._hostnameASCII, e)
except ValueError as e:
logger.warning(
'Ignoring error while verifying certificate '
'from host "{}" (exception: {})'.format(
self._hostnameASCII, repr(e)))
'from host "%s" (exception: %r)',
self._hostnameASCII, e)
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')

12
scrapy/core/downloader/webclient.py
View File

@ -1,13 +1,14 @@
import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.http import HTTPClient
from twisted.internet import defer, reactor
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.responsetypes import responsetypes
@ -32,6 +33,8 @@ def _parse(url):
and is ascii-only.
"""
url = url.strip()
if not re.match(r'^\w+://', url):
url = '//' + url
parsed = urlparse(url)
return _parsed_url_args(parsed)
@ -109,8 +112,8 @@ class ScrapyHTTPClientFactory(ClientFactory):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body)
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
@ -167,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
p.followRedirect = self.followRedirect
p.afterFoundGet = self.afterFoundGet
if self.timeout:
from twisted.internet import reactor
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p

Some files were not shown because too many files have changed in this diff Show More