RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into patch-1

This commit is contained in:
Andrey Rahmatullin 2022-01-21 13:24:51 +05:00 committed by GitHub
parent 431f6e7d90 9b8285d98a
commit 1608b1a7fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
390 changed files with 22195 additions and 9184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.bumpversion.cfg
View File

@ -1,8 +1,7 @@
[bumpversion]
current_version = 1.8.0
current_version = 2.5.0
commit = True
tag = True
tag_name = {new_version}
[bumpversion:file:scrapy/VERSION]

19
.flake8 Normal file
View File

@ -0,0 +1,19 @@
[flake8]
max-line-length = 119
ignore = W503
exclude =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
tests/sample_data/** binary

45
.github/workflows/checks.yml vendored Normal file
View File

@ -0,0 +1,45 @@
name: Checks
on: [push, pull_request]
jobs:
checks:
runs-on: ubuntu-18.04
strategy:
fail-fast: false
matrix:
include:
- python-version: "3.10"
env:
TOXENV: security
- python-version: "3.10"
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: pylint
TOX_PIP_VERSION: 20.3.3
- python-version: 3.6
env:
TOXENV: typing
- python-version: "3.10" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run check
env: ${{ matrix.env }}
run: |
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
pip install tox-pip-version
fi
pip install -U tox
tox

31
.github/workflows/publish.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: Publish
on: [push]
jobs:
publish:
runs-on: ubuntu-18.04
if: startsWith(github.event.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Check Tag
id: check-release-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
echo ::set-output name=release_tag::true
fi
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*

26
.github/workflows/tests-macos.yml vendored Normal file
View File

@ -0,0 +1,26 @@
name: macOS
on: [push, pull_request]
jobs:
tests:
runs-on: macos-10.15
strategy:
fail-fast: false
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run tests
run: |
pip install -U tox
tox -e py
- name: Upload coverage report
run: bash <(curl -s https://codecov.io/bash)

83
.github/workflows/tests-ubuntu.yml vendored Normal file
View File

@ -0,0 +1,83 @@
name: Ubuntu
on: [push, pull_request]
jobs:
tests:
runs-on: ubuntu-18.04
strategy:
fail-fast: false
matrix:
include:
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: asyncio
- python-version: pypy3
env:
TOXENV: pypy3
PYPY_VERSION: 3.6-v7.3.3
# pinned deps
- python-version: 3.6.12
env:
TOXENV: pinned
- python-version: 3.6.12
env:
TOXENV: asyncio-pinned
- python-version: pypy3
env:
TOXENV: pypy3-pinned
PYPY_VERSION: 3.6-v7.2.0
# extras
# extra-deps includes reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: extra-deps
TOX_PIP_VERSION: 20.3.3
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
run: |
sudo apt-get update
# libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
- name: Run tests
env: ${{ matrix.env }}
run: |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
pip install tox-pip-version
fi
pip install -U tox
tox
- name: Upload coverage report
run: bash <(curl -s https://codecov.io/bash)

42
.github/workflows/tests-windows.yml vendored Normal file
View File

@ -0,0 +1,42 @@
name: Windows
on: [push, pull_request]
jobs:
tests:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: 3.6
env:
TOXENV: windows-pinned
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: asyncio
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox

5
.gitignore vendored
View File

@ -14,7 +14,12 @@ htmlcov/
.coverage
.pytest_cache/
.coverage.*
coverage.*
test-output.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt
/tests/keys/localhost.key
# Windows
Thumbs.db

12
.readthedocs.yml
View File

@ -1,11 +1,17 @@
version: 2
formats: all
sphinx:
configuration: docs/conf.py
fail_on_warning: true
build:
os: ubuntu-20.04
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: "3.10" # Keep in sync with .github/workflows/checks.yml
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.7 # Keep in sync with .travis.yml
install:
- requirements: docs/requirements.txt
- path: .

65
.travis.yml
View File

@ -1,65 +0,0 @@
language: python
dist: xenial
branches:
only:
- master
- /^\d\.\d+$/
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
- env: TOXENV=security
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
- env: TOXENV=py35
python: 3.5
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py35-asyncio
python: 3.5.2
- env: TOXENV=py36
python: 3.6
- env: TOXENV=py37
python: 3.7
- env: TOXENV=py38
python: 3.8
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=py38-asyncio
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
- pip install -U tox twine wheel codecov
script: tox
after_success:
- codecov
notifications:
irc:
use_notice: true
skip_join: true
channels:
- irc.freenode.org#scrapy
cache:
directories:
- $HOME/.cache/pip
deploy:
provider: pypi
distributions: "sdist bdist_wheel"
user: scrapy
password:
secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
on:
tags: true
repo: scrapy/scrapy
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

4
AUTHORS
View File

@ -1,8 +1,8 @@
Scrapy was brought to life by Shane Evans while hacking a scraping framework
prototype for Mydeco (mydeco.com). It soon became maintained, extended and
improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to
bootstrap the project. In mid-2011, Scrapinghub became the new official
maintainer.
bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new
official maintainer.
Here is the list of the primary authors & contributors:

5
CODE_OF_CONDUCT.md
View File

@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at opensource@scrapinghub.com. All
reported by contacting the project team at opensource@zyte.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq

24
Makefile.buildbot
View File

@ -1,24 +0,0 @@
TRIAL := $(shell which trial)
BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
export PYTHONPATH=$(PWD)
test:
coverage run --branch $(TRIAL) --reporter=text tests
rm -rf htmlcov && coverage html
-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
build:
git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION
debchange -m -D unstable --force-distribution -v \
$$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \
"Automatic build"
debuild -us -uc -b
clean:
git checkout debian scrapy/VERSION
git clean -dfq
pypi:
umask 0022 && chmod -R a+rX . && python setup.py sdist upload
.PHONY: clean test build

28
README.rst
View File

@ -1,3 +1,6 @@
.. image:: /artwork/scrapy-logo.jpg
:width: 400px
======
Scrapy
======
@ -10,9 +13,17 @@ Scrapy
:target: https://pypi.python.org/pypi/Scrapy
:alt: Supported Python Versions
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
:target: https://travis-ci.org/scrapy/scrapy
:alt: Build Status
.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
:alt: Windows
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
:target: https://pypi.python.org/pypi/Scrapy
@ -34,13 +45,20 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other
contributors`_.
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
.. _Zyte: https://www.zyte.com/
Check the Scrapy homepage at https://scrapy.org for more information,
including a list of features.
Requirements
============
* Python 3.5+
* Python 3.6+
* Works on Linux, Windows, macOS, BSD
Install
@ -81,7 +99,7 @@ Please note that this project is released with a Contributor Code of Conduct
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
By participating in this project you agree to abide by its terms.
Please report unacceptable behavior to opensource@scrapinghub.com.
Please report unacceptable behavior to opensource@zyte.com.
Companies using Scrapy
======================

25
appveyor.yml
View File

@ -1,25 +0,0 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
- "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

49
conftest.py
View File

@ -1,6 +1,11 @@
from pathlib import Path
import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys
def _py_files(folder):
@ -12,14 +17,23 @@ collect_ignore = [
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
]
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
with open('tests/ignores.txt') as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
*_py_files("scrapy/core/http2"),
)
)
@pytest.fixture()
@ -38,6 +52,14 @@ def pytest_collection_modifyitems(session, config, items):
pass
def pytest_addoption(parser):
parser.addoption(
"--reactor",
default="default",
choices=["default", "asyncio"],
)
@pytest.fixture(scope='class')
def reactor_pytest(request):
if not request.cls:
@ -51,3 +73,18 @@ def reactor_pytest(request):
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
pytest.skip('This test is only run without --reactor=asyncio')
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
# Generate localhost certificate files, needed by some tests
generate_keys()

5
debian/changelog vendored
View File

@ -1,5 +0,0 @@
scrapy (0.11) unstable; urgency=low
* Initial release.
-- Scrapinghub Team <info@scrapinghub.com> Thu, 10 Jun 2010 17:24:02 -0300

1
debian/compat vendored
View File

@ -1 +0,0 @@
7

20
debian/control vendored
View File

@ -1,20 +0,0 @@
Source: scrapy
Section: python
Priority: optional
Maintainer: Scrapinghub Team <info@scrapinghub.com>
Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
Standards-Version: 3.8.4
Homepage: https://scrapy.org/
Package: scrapy
Architecture: all
Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
Recommends: python-setuptools
Conflicts: python-scrapy, scrapy-0.25
Provides: python-scrapy, scrapy-0.25
Description: Python web crawling and web scraping framework
Scrapy is a fast high-level web crawling and web scraping framework,
used to crawl websites and extract structured data from their pages.
It can be used for a wide range of purposes, from data mining to
monitoring and automated testing.

40
debian/copyright vendored
View File

@ -1,40 +0,0 @@
This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
It was downloaded from https://scrapy.org
Upstream Author: Scrapy Developers
Copyright: 2007-2013 Scrapy Developers
License: bsd
Copyright (c) Scrapy developers.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The Debian packaging is (C) 2010-2013, Scrapinghub <info@scrapinghub.com> and
is licensed under the BSD, see `/usr/share/common-licenses/BSD'.

1
debian/pyversions vendored
View File

@ -1 +0,0 @@
2.7

5
debian/rules vendored
View File

@ -1,5 +0,0 @@
#!/usr/bin/make -f
# -*- makefile -*-
%:
dh $@

2
debian/scrapy.docs vendored
View File

@ -1,2 +0,0 @@
README.rst
AUTHORS

2
debian/scrapy.install vendored
View File

@ -1,2 +0,0 @@
extras/scrapy_bash_completion etc/bash_completion.d/
extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy

1
debian/scrapy.lintian-overrides vendored
View File

@ -1 +0,0 @@
new-package-should-close-itp-bug

1
debian/scrapy.manpages vendored
View File

@ -1 +0,0 @@
extras/scrapy.1

2
docs/Makefile
View File

@ -8,7 +8,7 @@ PYTHON = python
SPHINXOPTS =
PAPER =
SOURCES =
SHELL = /bin/bash
SHELL = /usr/bin/env bash
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
-D latex_elements.papersize=$(PAPER) \

9
docs/README.rst
View File

@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you
need to install watchdog (``pip install watchdog``) and then use::
make watch
Alternative method using tox
----------------------------
To compile the documentation to HTML run the following command::
tox -e docs
Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.

2
docs/_ext/scrapydocs.py
View File

@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
def is_setting_index(node):
if node.tagname == 'index':
# index entries for setting directives look like:
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
return False

10
docs/_static/custom.css vendored Normal file
View File

@ -0,0 +1,10 @@
/* Move lists closer to their introducing paragraph */
.rst-content .section ol p, .rst-content .section ul p {
margin-bottom: 0px;
}
.rst-content p + ol, .rst-content p + ul {
margin-top: -18px; /* Compensates margin-top: 24px of p */
}
.rst-content dl p + ol, .rst-content dl p + ul {
margin-top: -6px; /* Compensates margin-top: 12px of p */
}

7
docs/_templates/layout.html vendored
View File

@ -3,14 +3,9 @@
{% block footer %}
{{ super() }}
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();
analytics.ready(function () {
ga('require', 'linker');
ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
ga('linker:autoLink', ['zyte.com']);
});
</script>
{% endblock %}

35
docs/conf.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
#
# Scrapy documentation build configuration file, created by
# sphinx-quickstart on Mon Nov 24 12:02:52 2008.
#
@ -51,7 +49,7 @@ master_doc = 'index'
# General information about the project.
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -102,6 +100,9 @@ exclude_trees = ['.build']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# List of Sphinx warnings that will not be raised
suppress_warnings = ['epub.unknown_project_files']
# Options for HTML output
# -----------------------
@ -121,7 +122,6 @@ html_theme = 'sphinx_rtd_theme'
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
@ -182,6 +182,10 @@ html_copy_source = True
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
html_css_files = [
'custom.css',
]
# Options for LaTeX output
# ------------------------
@ -280,8 +284,11 @@ coverage_ignore_pyobjects = [
# -------------------------------------
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cryptography' : ('https://cryptography.io/en/latest/', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
@ -295,3 +302,23 @@ intersphinx_mapping = {
# ------------------------------------
hoverxref_auto_ref = True
hoverxref_role_types = {
"class": "tooltip",
"confval": "tooltip",
"hoverxref": "tooltip",
"mod": "tooltip",
"ref": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
def setup(app):
app.connect('autodoc-skip-member', maybe_skip_member)
def maybe_skip_member(app, what, name, obj, skip, options):
if not skip:
# autodocs was generating a text "alias of" for the following members
# https://github.com/sphinx-doc/sphinx/issues/4422
return name in {'default_item_class', 'default_selector_class'}
return skip

8
docs/conftest.py
View File

@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.codeblock import CodeBlockParser
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
@ -21,7 +25,7 @@ def setup(namespace):
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
CodeBlockParser(future_imports=['print_function']),
PythonCodeBlockParser(future_imports=['print_function']),
skip,
],
pattern='*.rst',

23
docs/contributing.rst
View File

@ -108,6 +108,11 @@ Well-written patches should:
tox -e docs-coverage
* if you are removing deprecated code, first make sure that at least 1 year
(12 months) has passed since the release that introduced the deprecation.
See :ref:`deprecation-policy`.
.. _submitting-patches:
Submitting patches
@ -135,7 +140,7 @@ original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
not considered rude as long as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch
@ -155,6 +160,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
removal, etc) in separate commits from functional changes. This will make pull
requests easier to review and more likely to get merged.
.. _coding-style:
Coding style
============
@ -163,7 +171,7 @@ Scrapy:
* Unless otherwise specified, follow :pep:`8`.
* It's OK to use lines longer than 80 chars if it improves the code
* It's OK to use lines longer than 79 chars if it improves the code
readability.
* Don't put your name in the code you contribute; git provides enough
@ -191,6 +199,17 @@ In any case, if something is covered in a docstring, use the
documentation instead of duplicating the docstring in files within the
``docs/`` directory.
Documentation updates that cover new or modified features must use Sphinx’s
:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use
``VERSION`` as version, we will replace it with the actual version right before
the corresponding release. When we release a new major or minor version of
Scrapy, we remove these directives if they are older than 3 years.
Documentation about deprecated features must be removed as those features are
deprecated, so that new readers do not run into it. New deprecations and
deprecation removals are documented in the :ref:`release notes <news>`.
Tests
=====

91
docs/faq.rst
View File

@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
.. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
.. _faq-python-versions:
What Python versions does Scrapy support?
-----------------------------------------
Scrapy is supported under Python 3.5+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
Python 2 support was dropped in Scrapy 2.0.
.. note::
For Python 3 support on Windows, it is recommended to use
Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
Did Scrapy "steal" X from Django?
---------------------------------
@ -159,6 +145,41 @@ How can I make Scrapy consume less memory?
See previous question.
How can I prevent memory errors due to many allowed domains?
------------------------------------------------------------
If you have a spider with a long list of
:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider
replacing the default
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
less memory. For example:
- If your domain names are similar enough, use your own regular expression
instead joining the strings in
:attr:`~scrapy.Spider.allowed_domains` into a complex regular
expression.
- If you can `meet the installation requirements`_, use pyre2_ instead of
Python’s re_ to compile your URL-filtering regular expression. See
:issue:`1908`.
See also other suggestions at `StackOverflow`_.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation::
SPIDER_MIDDLEWARES = {
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
'myproject.middlewares.CustomOffsiteMiddleware': 500,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
.. _pyre2: https://github.com/andreasvc/pyre2
.. _re: https://docs.python.org/library/re.html
.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
Can I use Basic HTTP Authentication in my spiders?
--------------------------------------------------
@ -250,15 +271,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
To dump into a JSON file::
scrapy crawl myspider -o items.json
scrapy crawl myspider -O items.json
To dump into a CSV file::
scrapy crawl myspider -o items.csv
scrapy crawl myspider -O items.csv
To dump into a XML file::
scrapy crawl myspider -o items.xml
scrapy crawl myspider -O items.xml
For more information see :ref:`topics-feed-exports`
@ -329,6 +350,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items
You may need to remove namespaces. See :ref:`removing-namespaces`.
.. _faq-split-item:
How to split an item into multiple items in an item pipeline?
@ -342,15 +364,15 @@ method for this purpose. For example::
from copy import deepcopy
from scrapy.item import BaseItem
from itemadapter import is_item, ItemAdapter
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if isinstance(item, (BaseItem, dict)):
for _ in range(item['multiply_by']):
if is_item(item):
adapter = ItemAdapter(item)
for _ in range(adapter['multiply_by']):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?
@ -371,8 +393,33 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
.. _faq-stop-response-download:
How can I cancel the download of a given response?
--------------------------------------------------
In some situations, it might be useful to stop the download of a certain response.
For instance, sometimes you can determine whether or not you need the full contents
of a response by inspecting its headers or the first bytes of its body. In that case,
you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received`
or :class:`~scrapy.signals.headers_received` signals and raising a
:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the
:ref:`topics-stop-response-download` topic for additional information and examples.
Running ``runspider`` I get ``error: No spider found in file: <filename>``
--------------------------------------------------------------------------
This may happen if your Scrapy project has a spider module with a name that
conflicts with the name of one of the `Python standard library modules`_, such
as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
See :issue:`2680`.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _Python standard library modules: https://docs.python.org/py-modindex.html
.. _Python package: https://pypi.org/
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search

13
docs/index.rst
View File

@ -78,7 +78,6 @@ Basic concepts
topics/settings
topics/exceptions
:doc:`topics/commands`
Learn about the command-line tool used to manage your Scrapy project.
@ -165,6 +164,8 @@ Solving specific problems
topics/autothrottle
topics/benchmarking
topics/jobs
topics/coroutines
topics/asyncio
:doc:`faq`
Get answers to most frequently asked questions.
@ -205,6 +206,12 @@ Solving specific problems
:doc:`topics/jobs`
Learn how to pause and resume crawls for large spiders.
:doc:`topics/coroutines`
Use the :ref:`coroutine syntax <async>`.
:doc:`topics/asyncio`
Use :mod:`asyncio` and :mod:`asyncio`-powered libraries.
.. _extending-scrapy:
Extending Scrapy
@ -220,6 +227,7 @@ Extending Scrapy
topics/extensions
topics/api
topics/signals
topics/scheduler
topics/exporters
@ -241,6 +249,9 @@ Extending Scrapy
:doc:`topics/signals`
See all available signals and how to work with them.
:doc:`topics/scheduler`
Understand the scheduler component.
:doc:`topics/exporters`
Quickly export your scraped items to a file (XML, CSV, etc).

51
docs/intro/install.rst
View File

@ -4,12 +4,19 @@
Installation guide
==================
.. _faq-python-versions:
Supported Python versions
=========================
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
Installing Scrapy
=================
Scrapy runs on Python 3.5 or above
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
and macOS.
@ -23,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with::
pip install Scrapy
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
For more detailed and platform specifics instructions, as well as
troubleshooting information, read on.
@ -63,10 +70,9 @@ In case of any trouble related to these dependencies,
please refer to their respective installation instructions:
* `lxml installation`_
* `cryptography installation`_
* :doc:`cryptography installation <cryptography:installation>`
.. _lxml installation: https://lxml.de/installation.html
.. _cryptography installation: https://cryptography.io/en/latest/installation/
.. _intro-using-virtualenv:
@ -112,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
conda install -c conda-forge scrapy
To install Scrapy on Windows using ``pip``:
.. warning::
This installation method requires “Microsoft Visual C++” for installing some
Scrapy dependencies, which demands significantly more disk space than Anaconda.
#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer.
#. Run the Visual Studio Installer.
#. Under the Workloads section, select **C++ build tools**.
#. Check the installation details and make sure following packages are selected as optional components:
* **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) )
* **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0))
#. Install the Visual Studio Build Tools.
Now, you should be able to :ref:`install Scrapy <intro-install-scrapy>` using ``pip``.
.. _intro-install-ubuntu:
@ -207,8 +234,8 @@ For PyPy3, only Linux installation was tested.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
This means that these dependencies will be built during installation.
On macOS, you are likely to face an issue with building the Cryptography dependency. The
solution to this problem is described
On macOS, you are likely to face an issue with building the Cryptography
dependency. The solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
recommends (only needed when installing Scrapy). Installing on Linux has no special
@ -259,10 +286,10 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
.. _cryptography: https://cryptography.io/en/latest/
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
.. _setuptools: https://pypi.python.org/pypi/setuptools
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
.. _homebrew: https://brew.sh/
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
.. _conda-forge: https://conda-forge.org/

28
docs/intro/overview.rst
View File

@ -4,7 +4,7 @@
Scrapy at a glance
==================
Scrapy is an application framework for crawling web sites and extracting
Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
structured data which can be used for a wide range of useful applications, like
data mining, information processing or historical archival.
@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
if next_page is not None:
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.json
scrapy runspider quotes_spider.py -o quotes.jl
When this finishes you will have in the ``quotes.jl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
When this finishes you will have in the ``quotes.json`` file a list of the
quotes in JSON format, containing text and author, looking like this (reformatted
here for better readability)::
[{
"author": "Jane Austen",
"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
},
{
"author": "Groucho Marx",
"text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
},
{
"author": "Steve Martin",
"text": "\u201cA day without sunshine is like, you know, night.\u201d"
},
...]
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
{"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
{"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
...
What just happened?

86
docs/intro/tutorial.rst
View File

@ -25,16 +25,16 @@ Scrapy.
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
If you're new to programming and want to start with Python, the following books
may be useful to you:
may be useful to you:
* `Automate the Boring Stuff With Python`_
* `How To Think Like a Computer Scientist`_
* `How To Think Like a Computer Scientist`_
* `Learn Python 3 The Hard Way`_
* `Learn Python 3 The Hard Way`_
You can also take a look at `this list of Python resources for non-programmers`_,
as well as the `suggested resources in the learnpython-subreddit`_.
as well as the `suggested resources in the learnpython-subreddit`_.
.. _Python: https://www.python.org/
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents::
__init__.py
items.py # project items definition file
middlewares.py # project middlewares file
pipelines.py # project pipelines file
@ -78,7 +78,7 @@ Our first Spider
Spiders are classes that you define and that Scrapy uses to scrape information
from a website (or a group of websites). They must subclass
:class:`~scrapy.spiders.Spider` and define the initial requests to make,
:class:`~scrapy.Spider` and define the initial requests to make,
optionally how to follow links in the pages, and how to parse the downloaded
page content to extract data.
@ -101,32 +101,32 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
self.log('Saved file %s' % filename)
self.log(f'Saved file {filename}')
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
and defines some attributes and methods:
* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be
unique within a project, that is, you can't set the same name for different
Spiders.
* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of
* :meth:`~scrapy.Spider.start_requests`: must return an iterable of
Requests (you can return a list of requests or write a generator function)
which the Spider will begin to crawl from. Subsequent requests will be
generated successively from these initial requests.
* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
* :meth:`~scrapy.Spider.parse`: a method that will be called to handle
the response downloaded for each of the requests made. The response parameter
is an instance of :class:`~scrapy.http.TextResponse` that holds
the page content and has further helpful methods to handle it.
The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting
the scraped data as dicts and also finding new URLs to
follow and creating new requests (:class:`~scrapy.http.Request`) from them.
follow and creating new requests (:class:`~scrapy.Request`) from them.
How to run our spider
---------------------
@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs.
What just happened under the hood?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects
returned by the ``start_requests`` method of the Spider. Upon receiving a
response for each one, it instantiates :class:`~scrapy.http.Response` objects
and calls the callback method associated with the request (in this case, the
@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the
A shortcut to the start_requests method
---------------------------------------
Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method
that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs,
you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute
Instead of implementing a :meth:`~scrapy.Spider.start_requests` method
that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
import scrapy
@ -190,13 +190,13 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy
to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's
to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's
default callback method, which is called for requests without an explicitly
assigned callback.
@ -248,7 +248,7 @@ object:
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements
:class:`~scrapy.Selector` objects that wrap around XML/HTML elements
and allow you to run further queries to fine-grain the selection or extract the
data.
@ -277,9 +277,19 @@ As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
find any element matching the selection.
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
raise an :exc:`IndexError` exception if there are no results::
>>> response.css('noelement')[0].get()
Traceback (most recent call last):
...
IndexError: list index out of range
You might want to use ``.get()`` directly on the
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
if there are no results::
>>> response.css("noelement").get()
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
@ -287,8 +297,8 @@ to be scraped, you can at least get **some** data.
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
expressions`_:
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
:doc:`regular expressions <library/re>`:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
@ -305,7 +315,6 @@ with a selector (see :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
.. _regular expressions: https://docs.python.org/3/library/re.html
.. _Selector Gadget: https://selectorgadget.com/
@ -406,8 +415,6 @@ to get all of them:
from sys import version_info
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
@ -465,16 +472,15 @@ Storing the scraped data
The simplest way to store the scraped data is by using :ref:`Feed exports
<topics-feed-exports>`, with the following command::
scrapy crawl quotes -o quotes.json
scrapy crawl quotes -O quotes.json
That will generate an ``quotes.json`` file containing all scraped items,
That will generate a ``quotes.json`` file containing all scraped items,
serialized in `JSON`_.
For historic reasons, Scrapy appends to a given file instead of overwriting
its contents. If you run this command twice without removing the file
before the second time, you'll end up with a broken JSON file.
You can also use other formats, like `JSON Lines`_::
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
@ -674,7 +680,7 @@ the pagination links with the ``parse`` callback as we saw before.
Here we're passing callbacks to
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
arguments to make the code shorter; it also works for
:class:`~scrapy.http.Request`.
:class:`~scrapy.Request`.
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.
@ -705,7 +711,7 @@ Using spider arguments
You can provide command line arguments to your spiders by using the ``-a``
option when running them::
scrapy crawl quotes -o quotes-humor.json -a tag=humor
scrapy crawl quotes -O quotes-humor.json -a tag=humor
These arguments are passed to the Spider's ``__init__`` method and become
spider attributes by default.

1581
docs/news.rst
View File

File diff suppressed because it is too large Load Diff

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page
sphinx_rtd_theme
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx-rtd-theme>=0.5.2

8
docs/topics/api.rst
View File

@ -4,8 +4,6 @@
Core API
========
.. versionadded:: 0.15
This section documents the Scrapy core API, and it's intended for developers of
extensions and middlewares.
@ -31,7 +29,7 @@ how you :ref:`configure the downloader middlewares
.. class:: Crawler(spidercls, settings)
The Crawler object must be instantiated with a
:class:`scrapy.spiders.Spider` subclass and a
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
.. attribute:: settings
@ -91,7 +89,7 @@ how you :ref:`configure the downloader middlewares
provided while constructing the crawler, and it is created after the
arguments given in the :meth:`crawl` method.
.. method:: crawl(\*args, \**kwargs)
.. method:: crawl(*args, **kwargs)
Starts the crawler by instantiating its spider class with the given
``args`` and ``kwargs`` arguments, while setting the execution engine in
@ -198,7 +196,7 @@ SpiderLoader API
match the request's url against the domains of the spiders.
:param request: queried request
:type request: :class:`~scrapy.http.Request` instance
:type request: :class:`~scrapy.Request` instance
.. _topics-api-signals:

9
docs/topics/architecture.rst
View File

@ -67,7 +67,7 @@ this:
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
to crawl.
9. The process repeats (from step 1) until there are no more requests from the
9. The process repeats (from step 3) until there are no more requests from the
:ref:`Scheduler <component-scheduler>`.
Components
@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the
Scheduler
---------
The Scheduler receives requests from the engine and enqueues them for feeding
them later (also to the engine) when the engine requests them.
The :ref:`scheduler <topics-scheduler>` receives requests from the engine and
enqueues them for feeding them later (also to the engine) when the engine
requests them.
.. _component-downloader:
@ -104,7 +105,7 @@ Spiders
-------
Spiders are custom classes written by Scrapy users to parse responses and
extract items (aka scraped items) from them or additional requests to
extract :ref:`items <topics-items>` from them or additional requests to
follow. For more information see :ref:`topics-spiders`.
.. _component-pipelines:

83
docs/topics/asyncio.rst Normal file
View File

@ -0,0 +1,83 @@
.. _using-asyncio:
=======
asyncio
=======
.. versionadded:: 2.0
Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. _install-asyncio:
Installing the asyncio reactor
==============================
To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to
``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.
If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to
install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`::
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
Using custom asyncio loops
==========================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
use it instead of the default asyncio event loop.
.. _asyncio-await-dfd:
Windows-specific notes
======================
The Windows implementation of :mod:`asyncio` can use two event loop
implementations: :class:`~asyncio.SelectorEventLoop` (default before Python
3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop`
(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the
event loop class needs to be changed. Scrapy since VERSION does this
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor
by other means or use an older Scrapy version you need to call the following
code before installing the reactor::
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
You can put this in the same function that installs the reactor, if you do that
yourself, or in some code that runs before the reactor is installed, e.g.
``settings.py``.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
Awaiting on Deferreds
=====================
When the asyncio reactor isn't installed, you can await on Deferreds in the
coroutines directly. When it is installed, this is not possible anymore, due to
specifics of the Scrapy coroutine integration (the coroutines are wrapped into
:class:`asyncio.Future` objects, not into
:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
Futures. Scrapy provides two helpers for this:
.. autofunction:: scrapy.utils.defer.deferred_to_future
.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
.. tip:: If you need to use these functions in code that aims to be compatible
with lower versions of Scrapy that do not provide these functions,
down to Scrapy 2.0 (earlier versions do not support
:mod:`asyncio`), you can copy the implementation of these functions
into your own code.

2
docs/topics/autothrottle.rst
View File

@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies.
AUTOTHROTTLE_TARGET_CONCURRENCY
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 1.1
Default: ``1.0``
Average number of requests Scrapy should be sending in parallel to remote

7
docs/topics/benchmarking.rst
View File

@ -4,8 +4,6 @@
Benchmarking
============
.. versionadded:: 0.17
Scrapy comes with a simple benchmarking suite that spawns a local HTTP server
and crawls it at the maximum possible speed. The goal of this benchmarking is
to get an idea of how Scrapy performs in your hardware, in order to have a
@ -83,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which
results in slower crawl rates. How slower depends on how much your spider does
and how well it's written.
In the future, more cases will be added to the benchmarking suite to cover
other common scenarios.
Use scrapy-bench_ for more complex benchmarking.
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench

12
docs/topics/commands.rst
View File

@ -6,8 +6,6 @@
Command line tool
=================
.. versionadded:: 0.10
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
here as the "Scrapy tool" to differentiate it from the sub-commands, which we
just call "commands" or "Scrapy commands".
@ -468,7 +466,7 @@ Supported options:
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
response
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
@ -491,6 +489,10 @@ Supported options:
* ``--verbose`` or ``-v``: display information for each depth level
* ``--output`` or ``-o``: dump scraped items to a file
.. versionadded:: 2.3
.. skip: start
Usage example::
@ -562,8 +564,6 @@ and Platform info, which is useful for bug reports.
bench
-----
.. versionadded:: 0.17
* Syntax: ``scrapy bench``
* Requires project: *no*
@ -598,8 +598,6 @@ Example:
Register commands via setup.py entry points
-------------------------------------------
.. note:: This is an experimental feature, use with caution.
You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.

14
docs/topics/contracts.rst
View File

@ -4,8 +4,6 @@
Spiders Contracts
=================
.. versionadded:: 0.15
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@ -39,7 +37,7 @@ This callback is tested using three built-in contracts:
.. class:: CallbackKeywordArgumentsContract
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
attribute for the sample request. It must be a valid JSON dictionary.
::
@ -78,10 +76,10 @@ override three methods:
.. module:: scrapy.contracts
.. class:: Contract(method, \*args)
.. class:: Contract(method, *args)
:param method: callback function to which the contract is associated
:type method: function
:type method: collections.abc.Callable
:param args: list of arguments passed into the docstring (whitespace
separated)
@ -90,7 +88,7 @@ override three methods:
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
for request object. :class:`~scrapy.http.Request` is used by default,
for request object. :class:`~scrapy.Request` is used by default,
but this can be changed with the ``request_cls`` attribute.
If multiple contracts in chain have this attribute defined, the last one is used.
@ -136,7 +134,7 @@ Detecting check runs
====================
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use `os.environ`_ to perform any change to
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
your spiders or your settings when ``scrapy check`` is used::
import os
@ -148,5 +146,3 @@ your spiders or your settings when ``scrapy check`` is used::
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
pass # Do some scraper adjustments when a check is running
.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

110
docs/topics/coroutines.rst Normal file
View File

@ -0,0 +1,110 @@
==========
Coroutines
==========
.. versionadded:: 2.0
Scrapy has :ref:`partial support <coroutine-support>` for the
:ref:`coroutine syntax <async>`.
.. _coroutine-support:
Supported callables
===================
The following callables may be defined as coroutines using ``async def``, and
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.Request` callbacks.
.. note:: The callback output is not processed until the whole callback
finishes.
As a side effect, if the callback raises an exception, none of its
output is processed.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
- The
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`,
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`,
and
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
methods of
:ref:`downloader middlewares <topics-downloader-middleware-custom>`.
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
Usage
=====
There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
from itemadapter import ItemAdapter
class DbPipeline:
def _update_item(self, data, item):
adapter = ItemAdapter(item)
adapter['field'] = data
return item
def process_item(self, item, spider):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter['id'])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
from itemadapter import ItemAdapter
class DbPipeline:
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
adapter['field'] = await db.get_some_data(adapter['id'])
return item
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
class MySpiderDeferred(Spider):
# ...
async def parse(self, response):
additional_response = await treq.get('https://additional.url')
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
class MySpiderAsyncio(Spider):
# ...
async def parse(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
additional_data = await additional_response.text()
# ... use response and additional_data to yield items and requests
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
:mod:`asyncio` loop and to use them you need to
:doc:`enable asyncio support in Scrapy<asyncio>`.
.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
you need to :ref:`wrap them<asyncio-await-dfd>`.
Common use cases for asynchronous code include:
* requesting data from websites, databases and other services (in callbacks,
pipelines and middlewares);
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs

2
docs/topics/debug.rst
View File

@ -36,7 +36,7 @@ Consider the following Scrapy spider below::
Basically this is a simple spider which parses two pages of items (the
start_urls). Items also have a details page with additional information, so we
use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a
partially populated item.

32
docs/topics/deploy.rst
View File

@ -14,7 +14,7 @@ spiders come in.
Popular choices for deploying Scrapy spiders are:
* :ref:`Scrapyd <deploy-scrapyd>` (open source)
* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
* :ref:`Zyte Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
.. _deploy-scrapyd:
@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers.
.. _deploy-scrapy-cloud:
Deploying to Scrapy Cloud
=========================
Deploying to Zyte Scrapy Cloud
==============================
`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
the company behind Scrapy.
`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company
behind Scrapy.
Scrapy Cloud removes the need to setup and monitor servers
and provides a nice UI to manage spiders and review scraped items,
logs and stats.
Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a
nice UI to manage spiders and review scraped items, logs and stats.
To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
Please refer to the `Scrapy Cloud documentation`_ for more information.
To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line
tool.
Please refer to the `Zyte Scrapy Cloud documentation`_ for more information.
Scrapy Cloud is compatible with Scrapyd and one can switch between
Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between
them as needed - the configuration is read from the ``scrapy.cfg`` file
just like ``scrapyd-deploy``.
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
.. _shub: https://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: https://scrapinghub.com/
.. _shub: https://shub.readthedocs.io/en/latest/
.. _Zyte: https://zyte.com/
.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/
.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html

149
docs/topics/developer-tools.rst
View File

@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping
=================================================
Here is a general guide on how to use your browser's Developer Tools
to ease the scraping process. Today almost all browsers come with
to ease the scraping process. Today almost all browsers come with
built in `Developer Tools`_ and although we will use Firefox in this
guide, the concepts are applicable to any other browser.
guide, the concepts are applicable to any other browser.
In this guide we'll introduce the basic tools to use from a browser's
Developer Tools by scraping `quotes.toscrape.com`_.
@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things:
Inspecting a website
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
`quotes.toscrape.com`_-site.
On the site we have a total of ten quotes from various authors with specific
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
Instead of viewing the whole source code for the page, we can simply right click
Instead of viewing the whole source code for the page, we can simply right click
on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
In it you should see something like this:
@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
see each quote:
.. code-block:: html
@ -121,7 +121,7 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
@ -130,45 +130,45 @@ the `has-class-extension`_:
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
complex and by simply constructing an XPath with ``has-class("text")``
we were able to extract all quotes in one line.
we were able to extract all quotes in one line.
The `Inspector` has a lot of other helpful features, such as searching in the
The `Inspector` has a lot of other helpful features, such as searching in the
source code or directly scrolling to an element you selected. Let's demonstrate
a use case:
a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
the need to find an element visually but the ``Scroll into View`` function
can be quite useful on complex sites.
can be quite useful on complex sites.
Note that the search bar can also be used to search for and test CSS
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
.. _topics-network-tool:
The Network-tool
================
While scraping you may come across dynamic webpages where some parts
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
greatly facilitates this task. To demonstrate the Network-tool, let's
take a look at the page `quotes.toscrape.com/scroll`_.
take a look at the page `quotes.toscrape.com/scroll`_.
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the Scrapy shell:
.. skip: next
@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell:
(...)
>>> view(response)
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
.. image:: _images/network_01.png
:width: 777
@ -189,21 +189,21 @@ bar with the word ``Loading...``.
:alt: Response from quotes.toscrape.com/scroll
The ``view(response)`` command let's us view the response our
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
the login-button and the footer, but the quotes are missing. This
tells us that the quotes are being loaded from a different request
than ``quotes.toscrape/scroll``.
than ``quotes.toscrape/scroll``.
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
log is automatically cleared each time you navigate to a different
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
If we reload the page now, you'll see the log get populated with six
new requests.
new requests.
.. image:: _images/network_02.png
:width: 777
@ -212,31 +212,31 @@ new requests.
Here we see every request that has been made when reloading the page
and can inspect each request and its response. So let's find out
where our quotes are coming from:
where our quotes are coming from:
First click on the request with the name ``scroll``. On the right
First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address,
and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
on the request and open ``Open in new tab`` to get a better overview.
.. image:: _images/network_03.png
:width: 777
:height: 375
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
import scrapy
@ -255,17 +255,17 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
We iterate through the ``quotes`` and print out the ``quote["text"]``.
If the handy ``has_next`` element is ``true`` (try loading
If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
into our ``url``.
.. _requests-from-curl:
@ -274,7 +274,7 @@ In more complex websites, it could be difficult to easily reproduce the
requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
format, by right-clicking on each of them in the network tool and using the
:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
request::
from scrapy import Request
@ -289,11 +289,16 @@ request::
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments:
.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.

159
docs/topics/downloader-middleware.rst
View File

@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
middleware.
:meth:`process_request` should either: return ``None``, return a
:class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request`
:class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request`
object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`.
If it returns ``None``, Scrapy will continue processing this request, executing all
@ -88,7 +88,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
or the appropriate download function; it'll return that response. The :meth:`process_response`
methods of installed middleware is always called on every response.
If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
process_request methods and reschedule the returned request. Once the newly returned
request is performed, the appropriate middleware chain will be called on
the downloaded response.
@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
ignored and not logged (unlike other exceptions).
:param request: the request being processed
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_response(request, response, spider)
:meth:`process_response` should either: return a :class:`~scrapy.http.Response`
object, return a :class:`~scrapy.http.Request` object or
object, return a :class:`~scrapy.Request` object or
raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception.
If it returns a :class:`~scrapy.http.Response` (it could be the same given
response, or a brand-new one), that response will continue to be processed
with the :meth:`process_response` of the next middleware in the chain.
If it returns a :class:`~scrapy.http.Request` object, the middleware chain is
If it returns a :class:`~scrapy.Request` object, the middleware chain is
halted and the returned request is rescheduled to be downloaded in the future.
This is the same behavior as if a request is returned from :meth:`process_request`.
@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
exception, it is ignored and not logged (unlike other exceptions).
:param request: the request that originated the response
:type request: is a :class:`~scrapy.http.Request` object
:type request: is a :class:`~scrapy.Request` object
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_exception(request, exception, spider)
@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception)
:meth:`process_exception` should return: either ``None``,
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object.
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_exception` methods of installed middleware,
@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
method chain of installed middleware is started, and Scrapy won't bother calling
any other :meth:`process_exception` methods of middleware.
If it returns a :class:`~scrapy.http.Request` object, the returned request is
If it returns a :class:`~scrapy.Request` object, the returned request is
rescheduled to be downloaded in the future. This stops the execution of
:meth:`process_exception` methods of the middleware the same as returning a
response would.
:param request: the request that generated the exception
:type request: is a :class:`~scrapy.http.Request` object
:type request: is a :class:`~scrapy.Request` object
:param exception: the raised exception
:type exception: an ``Exception`` object
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
@ -202,6 +202,16 @@ CookiesMiddleware
sends them back on subsequent requests (from that spider), just like web
browsers do.
.. caution:: When non-UTF8 encoded byte sequences are passed to a
:class:`~scrapy.Request`, the ``CookiesMiddleware`` will log
a warning. Refer to :ref:`topics-logging-advanced-customization`
to customize the logging behaviour.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
The following settings can be used to configure the cookie middleware:
* :setting:`COOKIES_ENABLED`
@ -212,8 +222,6 @@ The following settings can be used to configure the cookie middleware:
Multiple cookie sessions per spider
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 0.15
There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
@ -250,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will
**not** be merged with the existing cookies.
For more detailed information see the ``cookies`` parameter in
:class:`~scrapy.http.Request`.
:class:`~scrapy.Request`.
.. setting:: COOKIES_DEBUG
@ -315,8 +323,21 @@ HttpAuthMiddleware
This middleware authenticates all requests generated from certain spiders
using `Basic access authentication`_ (aka. HTTP auth).
To enable HTTP authentication from certain spiders, set the ``http_user``
and ``http_pass`` attributes of those spiders.
To enable HTTP authentication for a spider, set the ``http_user`` and
``http_pass`` spider attributes to the authentication data and the
``http_auth_domain`` spider attribute to the domain which requires this
authentication (its subdomains will be also handled in the same way).
You can set ``http_auth_domain`` to ``None`` to enable the
authentication for all requests but you risk leaking your authentication
credentials to unrelated domains.
.. warning::
In previous Scrapy versions HttpAuthMiddleware sent the authentication
data with all requests, which is a security problem if the spider
makes requests to several different domains. Currently if the
``http_auth_domain`` attribute is not set, the middleware will use the
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example::
@ -326,6 +347,7 @@ HttpAuthMiddleware
http_user = 'someuser'
http_pass = 'somepass'
http_auth_domain = 'intranet.example.com'
name = 'intranet.example.com'
# .. rest of the spider code omitted ...
@ -470,8 +492,6 @@ DBM storage backend
.. class:: DbmCacheStorage
.. versionadded:: 0.13
A DBM_ storage backend is also available for the HTTP cache middleware.
By default, it uses the :mod:`dbm`, but you can change it with the
@ -495,7 +515,7 @@ defines the methods described below.
the :signal:`open_spider <spider_opened>` signal.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: close_spider(spider)
@ -503,27 +523,27 @@ defines the methods described below.
the :signal:`close_spider <spider_closed>` signal.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: retrieve_response(spider, request)
Return response if present in cache, or ``None`` otherwise.
:param spider: the spider which generated the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param request: the request to find cached response for
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
.. method:: store_response(spider, request, response)
Store the given response in the cache.
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param request: the corresponding request the spider generated
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param response: the response to store in the cache
:type response: :class:`~scrapy.http.Response` object
@ -544,15 +564,10 @@ settings:
HTTPCACHE_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.11
Default: ``False``
Whether the HTTP cache will be enabled.
.. versionchanged:: 0.11
Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache.
.. setting:: HTTPCACHE_EXPIRATION_SECS
HTTPCACHE_EXPIRATION_SECS
@ -565,9 +580,6 @@ Expiration time for cached requests, in seconds.
Cached requests older than this time will be re-downloaded. If zero, cached
requests will never expire.
.. versionchanged:: 0.11
Before 0.11, zero meant cached requests always expire.
.. setting:: HTTPCACHE_DIR
HTTPCACHE_DIR
@ -584,8 +596,6 @@ project data dir. For more info see: :ref:`topics-project-structure`.
HTTPCACHE_IGNORE_HTTP_CODES
^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``[]``
Don't cache response with these HTTP codes.
@ -604,8 +614,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade
HTTPCACHE_IGNORE_SCHEMES
^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``['file']``
Don't cache responses with these URI schemes.
@ -624,8 +632,6 @@ The class which implements the cache storage backend.
HTTPCACHE_DBM_MODULE
^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
@ -636,8 +642,6 @@ The database module to use in the :ref:`DBM storage backend
HTTPCACHE_POLICY
^^^^^^^^^^^^^^^^
.. versionadded:: 0.18
Default: ``'scrapy.extensions.httpcache.DummyPolicy'``
The class which implements the cache policy.
@ -647,8 +651,6 @@ The class which implements the cache policy.
HTTPCACHE_GZIP
^^^^^^^^^^^^^^
.. versionadded:: 1.0
Default: ``False``
If enabled, will compress all cached data with gzip.
@ -659,8 +661,6 @@ This setting is specific to the Filesystem backend.
HTTPCACHE_ALWAYS_STORE
^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``False``
If enabled, will cache pages unconditionally.
@ -679,8 +679,6 @@ responses you feed to the cache middleware.
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``[]``
List of Cache-Control directives in responses to be ignored.
@ -705,11 +703,14 @@ HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ responses,
provided `brotlipy`_ is installed.
This middleware also supports decoding `brotli-compressed`_ as well as
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.org/project/brotlipy/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -730,16 +731,14 @@ HttpProxyMiddleware
.. module:: scrapy.downloadermiddlewares.httpproxy
:synopsis: Http Proxy Middleware
.. versionadded:: 0.8
.. reqmeta:: proxy
.. class:: HttpProxyMiddleware
This middleware sets the HTTP proxy to use for requests, by setting the
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
``proxy`` meta value for :class:`~scrapy.Request` objects.
Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
Like the Python standard library module :mod:`urllib.request`, it obeys
the following environment variables:
* ``http_proxy``
@ -751,9 +750,6 @@ HttpProxyMiddleware
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
environment variables, and it will also ignore ``no_proxy`` environment variable.
.. _urllib: https://docs.python.org/2/library/urllib.html
.. _urllib2: https://docs.python.org/2/library/urllib2.html
RedirectMiddleware
------------------
@ -767,12 +763,12 @@ RedirectMiddleware
.. reqmeta:: redirect_urls
The urls which the request goes through (while being redirected) can be found
in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
in the ``redirect_urls`` :attr:`Request.meta <scrapy.Request.meta>` key.
.. reqmeta:: redirect_reasons
The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
``redirect_reasons`` :attr:`Request.meta <scrapy.Request.meta>` key. For
example: ``[301, 302, 307, 'meta refresh']``.
The format of a reason depends on the middleware that handled the corresponding
@ -788,7 +784,7 @@ settings (see the settings documentation for more info):
.. reqmeta:: dont_redirect
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_redirect``
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_redirect``
key set to True, the request will be ignored by this middleware.
If you want to handle some redirect status codes in your spider, you can
@ -801,7 +797,7 @@ responses (and pass them through to your spider) you can do this::
handle_httpstatus_list = [301, 302]
The ``handle_httpstatus_list`` key of :attr:`Request.meta
<scrapy.http.Request.meta>` can also be used to specify which response codes to
<scrapy.Request.meta>` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key
``handle_httpstatus_all`` to ``True`` if you want to allow any response code
for a request.
@ -815,8 +811,6 @@ RedirectMiddleware settings
REDIRECT_ENABLED
^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Redirect middleware will be enabled.
@ -829,6 +823,7 @@ REDIRECT_MAX_TIMES
Default: ``20``
The maximum number of redirections that will be followed for a single request.
After this maximum, the request's response is returned as is.
MetaRefreshMiddleware
---------------------
@ -857,8 +852,6 @@ MetaRefreshMiddleware settings
METAREFRESH_ENABLED
^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.17
Default: ``True``
Whether the Meta Refresh middleware will be enabled.
@ -872,6 +865,10 @@ Default: ``[]``
Meta tags within these tags are ignored.
.. versionchanged:: 2.0
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``['script', 'noscript']`` to ``[]``.
.. setting:: METAREFRESH_MAXDELAY
METAREFRESH_MAXDELAY
@ -906,9 +903,14 @@ settings (see the settings documentation for more info):
.. reqmeta:: dont_retry
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
set to True, the request will be ignored by this middleware.
To retry requests from a spider callback, you can use the
:func:`get_retry_request` function:
.. autofunction:: get_retry_request
RetryMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~
@ -917,8 +919,6 @@ RetryMiddleware Settings
RETRY_ENABLED
^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Retry middleware will be enabled.
@ -933,7 +933,7 @@ Default: ``2``
Maximum number of times to retry, in addition to the first download.
Maximum number of retries can also be specified per-request using
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.Request.meta>`.
When initialized, the :reqmeta:`max_retry_times` meta key takes higher
precedence over the :setting:`RETRY_TIMES` setting.
@ -951,6 +951,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. _topics-dlmw-robots:
@ -988,7 +1000,7 @@ RobotsTxtMiddleware
.. reqmeta:: dont_obey_robotstxt
If :attr:`Request.meta <scrapy.http.Request.meta>` has
If :attr:`Request.meta <scrapy.Request.meta>` has
``dont_obey_robotstxt`` key set to True
the request will be ignored by this middleware even if
:setting:`ROBOTSTXT_OBEY` is enabled.
@ -1032,8 +1044,7 @@ Scrapy uses this parser by default.
RobotFileParser
~~~~~~~~~~~~~~~
Based on `RobotFileParser
<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
Based on :class:`~urllib.robotparser.RobotFileParser`:
* is Python's built-in robots.txt_ parser
@ -1073,9 +1084,13 @@ In order to use this parser:
* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
.. warning:: `Upstream issue #122
<https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
* Set :setting:`ROBOTSTXT_PARSER` setting to
``scrapy.robotstxt.ReppyRobotParser``
.. _rerp-parser:
Robotexclusionrulesparser
@ -1173,8 +1188,6 @@ AjaxCrawlMiddleware Settings
AJAXCRAWL_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.21
Default: ``False``
Whether the AjaxCrawlMiddleware will be enabled. You may want to

82
docs/topics/dynamic-content.rst
View File

@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the
information can be found in the response they get.
If they get a response with the desired data, modify your Scrapy
:class:`~scrapy.http.Request` to match that of the other HTTP client. For
:class:`~scrapy.Request` to match that of the other HTTP client. For
example, try using the same user-agent string (:setting:`USER_AGENT`) or the
same :attr:`~scrapy.http.Request.headers`.
same :attr:`~scrapy.Request.headers`.
If they also get a response without the desired data, you’ll need to take
steps to make your request more similar to that of the web browser. See
@ -81,14 +81,14 @@ Use the :ref:`network tool <topics-network-tool>` of your web browser to see
how your web browser performs the desired request, and try to reproduce that
request with Scrapy.
It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
method and URL. However, you may also need to reproduce the body, headers and
form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
form parameters (see :class:`~scrapy.FormRequest`) of that request.
As all major browsers allow to export the requests in `cURL
<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
:class:`~scrapy.http.Request` from a cURL command. To get more information
:meth:`~scrapy.Request.from_curl()` to generate an equivalent
:class:`~scrapy.Request` from a cURL command. To get more information
visit :ref:`request from curl <requests-from-curl>` inside the network
tool section.
@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is
probably not your request, but the target server. The target server might be
buggy, overloaded, or :ref:`banning <bans>` some of your requests.
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
.. _topics-handling-response-formats:
Handling different response formats
@ -115,14 +118,14 @@ data from it depends on the type of response:
- If the response is HTML or XML, use :ref:`selectors
<topics-selectors>` as usual.
- If the response is JSON, use `json.loads`_ to load the desired data from
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
data = json.loads(response.text)
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a
:class:`~scrapy.selector.Selector` and then
:class:`~scrapy.Selector` and then
:ref:`use it <topics-selectors>` as usual::
selector = Selector(data['html'])
@ -130,8 +133,9 @@ data from it depends on the type of response:
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
- If the response is CSS, use a `regular expression`_ to extract the desired
data from :attr:`response.text <scrapy.http.TextResponse.text>`.
- If the response is CSS, use a :doc:`regular expression <library/re>` to
extract the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`.
.. _topics-parsing-images:
@ -168,8 +172,9 @@ JavaScript code:
Once you have a string with the JavaScript code, you can extract the desired
data from it:
- You might be able to use a `regular expression`_ to extract the desired
data in JSON format, which you can then parse with `json.loads`_.
- You might be able to use a :doc:`regular expression <library/re>` to
extract the desired data in JSON format, which you can then parse with
:func:`json.loads`.
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
@ -179,6 +184,18 @@ data from it:
>>> json.loads(json_data)
{'field': 'value'}
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
For example, if the JavaScript code contains
``var data = {field: "value", secondField: "second value"};``
you can extract that data as follows:
>>> import chompjs
>>> javascript = response.css('script::text').get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
@ -229,25 +246,46 @@ Using a headless browser
========================
A `headless browser`_ is a special web browser that provides an API for
automation.
automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
The easiest way to use a headless browser with Scrapy is to use Selenium_,
along with `scrapy-selenium`_ for seamless integration.
One such library is `playwright-python`_ (an official Python port of `playwright`_).
The following is a simple snippet to illustrate its usage within a Scrapy spider::
import scrapy
from playwright.async_api import async_playwright
class PlaywrightSpider(scrapy.Spider):
name = "playwright"
start_urls = ["data:,"] # avoid using the default Scrapy downloader
async def parse(self, response):
async with async_playwright() as pw:
browser = await pw.chromium.launch()
page = await browser.new_page()
await page.goto("https:/example.org")
title = await page.title()
return {"title": title}
However, using `playwright-python`_ directly as in the above example
circumvents most of the Scrapy components (middlewares, dupefilter, etc).
We recommend using `scrapy-playwright`_ for a better integration.
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _Splash: https://github.com/scrapinghub/splash
.. _chompjs: https://github.com/Nykakin/chompjs
.. _curl: https://curl.haxx.se/
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _js2xml: https://github.com/scrapinghub/js2xml
.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
.. _playwright-python: https://github.com/microsoft/playwright-python
.. _playwright: https://github.com/microsoft/playwright
.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
.. _pytesseract: https://github.com/madmaze/pytesseract
.. _regular expression: https://docs.python.org/3/library/re.html
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _Selenium: https://www.selenium.dev/
.. _Splash: https://github.com/scrapinghub/splash
.. _tabula-py: https://github.com/chezou/tabula-py
.. _wget: https://www.gnu.org/software/wget/
.. _wgrep: https://github.com/stav/wgrep
.. _wgrep: https://github.com/stav/wgrep

18
docs/topics/email.rst
View File

@ -7,7 +7,7 @@ Sending e-mail
.. module:: scrapy.mail
:synopsis: Email sending facility
Although Python makes sending e-mails relatively easy via the `smtplib`_
Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
library, Scrapy provides its own facility for sending e-mails which is very
easy to use and it's implemented using :doc:`Twisted non-blocking IO
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
@ -15,8 +15,6 @@ IO of the crawler. It also provides a simple API for sending attachments and
it's very easy to configure, with a few :ref:`settings
<topics-email-settings>`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
Quick example
=============
@ -64,10 +62,10 @@ rest of the framework.
:type smtpport: int
:param smtptls: enforce using SMTP STARTTLS
:type smtptls: boolean
:type smtptls: bool
:param smtpssl: enforce using a secure SSL connection
:type smtpssl: boolean
:type smtpssl: bool
.. classmethod:: from_settings(settings)
@ -81,14 +79,14 @@ rest of the framework.
Send email to the given recipients.
:param to: the e-mail recipients
:type to: str or list of str
:param to: the e-mail recipients as a string or as a list of strings
:type to: str or list
:param subject: the subject of the e-mail
:type subject: str
:param cc: the e-mails to CC
:type cc: str or list of str
:param cc: the e-mails to CC as a string or as a list of strings
:type cc: str or list
:param body: the e-mail body
:type body: str
@ -98,7 +96,7 @@ rest of the framework.
appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
attachment and ``file_object`` is a readable file object with the
contents of the attachment
:type attachs: iterable
:type attachs: collections.abc.Iterable
:param mimetype: the MIME type of the e-mail
:type mimetype: str

58
docs/topics/exceptions.rst
View File

@ -14,13 +14,6 @@ Built-in Exceptions reference
Here's a list of all exceptions included in Scrapy and their usage.
DropItem
--------
.. exception:: DropItem
The exception that must be raised by item pipeline stages to stop processing an
Item. For more information see :ref:`topics-item-pipeline`.
CloseSpider
-----------
@ -47,6 +40,14 @@ DontCloseSpider
This exception can be raised in a :signal:`spider_idle` signal handler to
prevent the spider from being closed.
DropItem
--------
.. exception:: DropItem
The exception that must be raised by item pipeline stages to stop processing an
Item. For more information see :ref:`topics-item-pipeline`.
IgnoreRequest
-------------
@ -63,10 +64,10 @@ NotConfigured
This exception can be raised by some components to indicate that they will
remain disabled. Those components include:
* Extensions
* Item pipelines
* Downloader middlewares
* Spider middlewares
- Extensions
- Item pipelines
- Downloader middlewares
- Spider middlewares
The exception must be raised in the component's ``__init__`` method.
@ -77,3 +78,38 @@ NotSupported
This exception is raised to indicate an unsupported feature.
StopDownload
-------------
.. versionadded:: 2.2
.. exception:: StopDownload(fail=True)
Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signal handler to indicate that no further bytes should be downloaded for a response.
The ``fail`` boolean parameter controls which method will handle the resulting
response:
* If ``fail=True`` (default), the request errback is called. The response object is
available as the ``response`` attribute of the ``StopDownload`` exception,
which is in turn stored as the ``value`` attribute of the received
:class:`~twisted.python.failure.Failure` object. This means that in an errback
defined as ``def errback(self, failure)``, the response can be accessed though
``failure.value.response``.
* If ``fail=False``, the request callback is called instead.
In both cases, the response could have its body truncated: the body contains
all bytes received up until the exception is raised, including the bytes
received in the signal handler that raises the exception. Also, the response
object is marked with ``"download_stopped"`` in its :attr:`Response.flags`
attribute.
.. note:: ``fail`` is a keyword-only parameter, i.e. raising
``StopDownload(False)`` or ``StopDownload(True)`` will raise
a :class:`TypeError`.
See the documentation for the :class:`~scrapy.signals.bytes_received` and
:class:`~scrapy.signals.headers_received` signals
and the :ref:`topics-stop-response-download` topic for additional information and examples.

109
docs/topics/exporters.rst
View File

@ -40,26 +40,29 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
from itemadapter import ItemAdapter
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline(object):
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
def open_spider(self, spider):
self.year_to_exporter = {}
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
for exporter, xml_file in self.year_to_exporter.values():
exporter.finish_exporting()
xml_file.close()
def _exporter_for_item(self, item):
year = item['year']
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
exporter = XmlItemExporter(f)
xml_file = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(xml_file)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
return self.year_to_exporter[year]
self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0]
def process_item(self, item, spider):
exporter = self._exporter_for_item(item)
@ -87,7 +90,7 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
If you use :class:`~.Item` you can declare a serializer in the
If you use :class:`~scrapy.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
@ -96,7 +99,7 @@ Example::
import scrapy
def serialize_price(value):
return '$ %s' % str(value)
return f'$ {str(value)}'
class Product(scrapy.Item):
name = scrapy.Field()
@ -119,9 +122,9 @@ Example::
class ProductXmlExporter(XmlItemExporter):
def serialize_field(self, field, name, value):
if field == 'price':
return '$ %s' % str(value)
return super(Product, self).serialize_field(field, name, value)
if name == 'price':
return f'$ {str(value)}'
return super().serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -137,7 +140,7 @@ output examples, which assume you're exporting these two items::
BaseItemExporter
----------------
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
@ -148,6 +151,9 @@ BaseItemExporter
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
.. versionadded:: 2.0
The *dont_fail* parameter.
.. method:: export_item(item)
Exports the given item. This method must be implemented in subclasses.
@ -161,12 +167,12 @@ BaseItemExporter
By default, this method looks for a serializer :ref:`declared in the item
field <topics-exporters-serializers>` and returns the result of applying
that serializer to the value. If no serializer is found, it returns the
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
value unchanged.
:param field: the field being serialized. If a raw dict is being
exported (not :class:`~.Item`) *field* value is an empty dict.
:type field: :class:`~scrapy.item.Field` object or an empty dict
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
:class:`dict`.
:type field: :class:`~scrapy.Field` object or a :class:`dict` instance
:param name: the name of the field being serialized
:type name: str
@ -189,14 +195,17 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or None if you
want to export all fields. Defaults to None.
A list with the name of the fields that will be exported, or ``None`` if
you want to export all fields. Defaults to ``None``.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Some exporters may require fields_to_export list in order to export the
data properly when spiders return dicts (not :class:`~Item` instances).
When using :ref:`item objects <item-types>` that do not expose all their
possible fields, exporters that do not support exporting a different
subset of fields per item will only export the fields found in the first
item exported. Use ``fields_to_export`` to define all the fields to be
exported.
.. attribute:: export_empty_fields
@ -208,10 +217,7 @@ BaseItemExporter
.. attribute:: encoding
The encoding that will be used to encode unicode values. This only
affects unicode values (which are always serialized to str using this
encoding). Other value types are passed unchanged to the specific
serialization library.
The output character encoding.
.. attribute:: indent
@ -233,9 +239,9 @@ PythonItemExporter
XmlItemExporter
---------------
.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs)
.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
Exports Items in XML format to the specified file object.
Exports items in XML format to the specified file object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -287,9 +293,9 @@ XmlItemExporter
CsvItemExporter
---------------
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs)
Exports Items in CSV format to the given file-like object. If the
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
@ -300,15 +306,20 @@ CsvItemExporter
:param include_headers_line: If enabled, makes the exporter output a header
line with the field names taken from
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
:type include_headers_line: boolean
:type include_headers_line: bool
:param join_multivalued: The char (or chars) that will be used for joining
multi-valued fields, if found.
:type include_headers_line: str
:param errors: The optional string that specifies how encoding and decoding
errors are to be handled. For more information see
:class:`io.TextIOWrapper`.
:type errors: str
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
:func:`csv.writer` function, so you can use any :func:`csv.writer` function
argument to customize this exporter.
A typical output of this exporter would be::
@ -317,14 +328,12 @@ CsvItemExporter
Color TV,1200
DVD player,200
.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
PickleItemExporter
------------------
.. class:: PickleItemExporter(file, protocol=0, \**kwargs)
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
Exports Items in pickle format to the given file-like object.
Exports items in pickle format to the given file-like object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -332,21 +341,19 @@ PickleItemExporter
:param protocol: The pickle protocol to use.
:type protocol: int
For more information, refer to the `pickle module documentation`_.
For more information, see :mod:`pickle`.
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
PprintItemExporter
------------------
.. class:: PprintItemExporter(file, \**kwargs)
.. class:: PprintItemExporter(file, **kwargs)
Exports Items in pretty print format to the specified file object.
Exports items in pretty print format to the specified file object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -364,13 +371,13 @@ PprintItemExporter
JsonItemExporter
----------------
.. class:: JsonItemExporter(file, \**kwargs)
.. class:: JsonItemExporter(file, **kwargs)
Exports Items in JSON format to the specified file-like object, writing all
Exports items in JSON format to the specified file-like object, writing all
objects as a list of objects. The additional ``__init__`` method arguments are
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -390,18 +397,16 @@ JsonItemExporter
stream-friendly format, consider using :class:`JsonLinesItemExporter`
instead, or splitting the output in multiple chunks.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
JsonLinesItemExporter
---------------------
.. class:: JsonLinesItemExporter(file, \**kwargs)
.. class:: JsonLinesItemExporter(file, **kwargs)
Exports Items in JSON format to the specified file-like object, writing one
Exports items in JSON format to the specified file-like object, writing one
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
``__init__`` method argument to customize this exporter.
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -414,8 +419,6 @@ JsonLinesItemExporter
Unlike the one produced by :class:`JsonItemExporter`, the format produced by
this exporter is well suited for serializing large amounts of data.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
MarshalItemExporter
-------------------

29
docs/topics/extensions.rst
View File

@ -7,8 +7,7 @@ Extensions
The extensions framework provides a mechanism for inserting your own
custom functionality into Scrapy.
Extensions are just regular classes that are instantiated at Scrapy startup,
when extensions are initialized.
Extensions are just regular classes.
Extension settings
==================
@ -27,8 +26,8 @@ Loading & activating extensions
===============================
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
code must be performed in the class ``__init__`` method.
instance of the extension class per spider being run. All the extension
initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@ -107,7 +106,7 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging(object):
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count
@ -257,6 +256,12 @@ settings:
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
.. note::
When a certain closing condition is met, requests which are
currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS`
requests) are still processed.
.. setting:: CLOSESPIDER_TIMEOUT
CLOSESPIDER_TIMEOUT
@ -279,8 +284,6 @@ Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``.
Requests which are currently in the downloader queue (up to
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
If zero (or non set), spiders won't be closed by number of passed items.
.. setting:: CLOSESPIDER_PAGECOUNT
@ -288,8 +291,6 @@ If zero (or non set), spiders won't be closed by number of passed items.
CLOSESPIDER_PAGECOUNT
"""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of responses to crawl. If the spider
@ -302,8 +303,6 @@ number of crawled responses.
CLOSESPIDER_ERRORCOUNT
""""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of errors to receive before
@ -324,6 +323,11 @@ domain has finished scraping, including the Scrapy stats collected. The email
will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
setting.
Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a
full list of parameters, including examples on how to instantiate
:class:`~scrapy.mail.MailSender` and use mail settings, see
:ref:`topics-email`.
.. module:: scrapy.extensions.debug
:synopsis: Extensions for debugging Scrapy
@ -364,7 +368,7 @@ Debugger extension
.. class:: Debugger
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process when a `SIGUSR2`_
signal is received. After the debugger is exited, the Scrapy process continues
running normally.
@ -372,5 +376,4 @@ For more info see `Debugging in Python`_.
This extension only works on POSIX-compliant platforms (i.e. not Windows).
.. _Python debugger: https://docs.python.org/2/library/pdb.html
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

527
docs/topics/feed-exports.rst
View File

@ -4,15 +4,13 @@
Feed exports
============
.. versionadded:: 0.10
One of the most frequently required features when implementing scrapers is
being able to store the scraped data properly and, quite often, that means
generating an "export file" with the scraped data (commonly called "export
feed") to be consumed by other systems.
Scrapy provides this functionality out of the box with the Feed Exports, which
allows you to generate a feed with the scraped items, using multiple
allows you to generate feeds with the scraped items, using multiple
serialization formats and storage backends.
.. _topics-feed-format:
@ -23,10 +21,10 @@ Serialization formats
For serializing the scraped data, the feed exports use the :ref:`Item exporters
<topics-exporters>`. These formats are supported out of the box:
* :ref:`topics-feed-format-json`
* :ref:`topics-feed-format-jsonlines`
* :ref:`topics-feed-format-csv`
* :ref:`topics-feed-format-xml`
- :ref:`topics-feed-format-json`
- :ref:`topics-feed-format-jsonlines`
- :ref:`topics-feed-format-csv`
- :ref:`topics-feed-format-xml`
But you can also extend the supported format through the
:setting:`FEED_EXPORTERS` setting.
@ -36,54 +34,58 @@ But you can also extend the supported format through the
JSON
----
* :setting:`FEED_FORMAT`: ``json``
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
- Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
- See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
.. _topics-feed-format-jsonlines:
JSON lines
----------
* :setting:`FEED_FORMAT`: ``jsonlines``
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
.. _topics-feed-format-csv:
CSV
---
* :setting:`FEED_FORMAT`: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
.. _topics-feed-format-xml:
XML
---
* :setting:`FEED_FORMAT`: ``xml``
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
- Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
.. _topics-feed-format-pickle:
Pickle
------
* :setting:`FEED_FORMAT`: ``pickle``
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
- Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
.. _topics-feed-format-marshal:
Marshal
-------
* :setting:`FEED_FORMAT`: ``marshal``
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
.. _topics-feed-storage:
@ -91,16 +93,17 @@ Marshal
Storages
========
When using the feed exports you define where to store the feed using a URI_
(through the :setting:`FEED_URI` setting). The feed exports supports multiple
When using the feed exports you define where to store the feed using one or multiple URIs_
(through the :setting:`FEEDS` setting). The feed exports supports multiple
storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-stdout`
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
@ -115,8 +118,8 @@ Storage URI parameters
The storage URI can also contain parameters that get replaced when the feed is
being created. These parameters are:
* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
* ``%(name)s`` - gets replaced by the spider name
- ``%(time)s`` - gets replaced by a timestamp when the feed is being created
- ``%(name)s`` - gets replaced by the spider name
Any other named parameter gets replaced by the spider attribute of the same
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
@ -124,13 +127,16 @@ attribute the moment the feed is being created.
Here are some examples to illustrate:
* Store in FTP using one directory per spider:
- Store in FTP using one directory per spider:
* ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
- ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
* Store in S3 using one directory per spider:
- Store in S3 using one directory per spider:
* ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
they can also be used as storage URI parameters.
.. _topics-feed-storage-backends:
@ -145,9 +151,9 @@ Local filesystem
The feeds are stored in the local filesystem.
* URI scheme: ``file``
* Example URI: ``file:///tmp/export.csv``
* Required external libraries: none
- URI scheme: ``file``
- Example URI: ``file:///tmp/export.csv``
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
@ -160,15 +166,18 @@ FTP
The feeds are stored in a FTP server.
* URI scheme: ``ftp``
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
- URI scheme: ``ftp``
- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
- Required external libraries: none
FTP supports two different connection modes: `active or passive
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-s3:
S3
@ -176,23 +185,61 @@ S3
The feeds are stored on `Amazon S3`_.
* URI scheme: ``s3``
* Example URIs:
- URI scheme: ``s3``
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Example URIs:
* Required external libraries: `botocore`_
- ``s3://mybucket/path/to/export.csv``
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_)
You can also define a custom ACL for exported feeds using this setting:
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-gcs:
Google Cloud Storage (GCS)
--------------------------
.. versionadded:: 2.3
The feeds are stored on `Google Cloud Storage`_.
- URI scheme: ``gs``
- Example URIs:
- ``gs://mybucket/path/to/export.csv``
- Required external libraries: `google-cloud-storage`_.
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
* :setting:`FEED_STORAGE_S3_ACL`
.. _topics-feed-storage-stdout:
@ -201,9 +248,126 @@ Standard output
The feeds are written to the standard output of the Scrapy process.
* URI scheme: ``stdout``
* Example URI: ``stdout:``
* Required external libraries: none
- URI scheme: ``stdout``
- Example URI: ``stdout:``
- Required external libraries: none
.. _delayed-file-delivery:
Delayed file delivery
---------------------
As indicated above, some of the described storage backends use delayed file
delivery.
These storage backends do not upload items to the feed URI as those items are
scraped. Instead, Scrapy writes items into a temporary local file, and only
once all the file contents have been written (i.e. at the end of the crawl) is
that file uploaded to the feed URI.
If you want item delivery to start earlier when using one of these storage
backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
in multiple files, with the specified maximum item count per file. That way, as
soon as a file reaches the maximum item count, that file is delivered to the
feed URI, allowing item delivery to start way before the end of the crawl.
.. _item-filter:
Item filtering
==============
.. versionadded:: VERSION
You can filter items that you want to allow for a particular feed by using the
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
the specified types will be added to the feed.
The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`
class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
method ``accepts`` and taking ``feed_options`` as an argument.
For instance::
class MyCustomFilter:
def __init__(self, feed_options):
self.feed_options = feed_options
def accepts(self, item):
if "field1" in item and item["field1"] == "expected_data":
return True
return False
You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.
See :setting:`FEEDS` for examples.
ItemFilter
----------
.. autoclass:: scrapy.extensions.feedexport.ItemFilter
:members:
.. _post-processing:
Post-Processing
===============
.. versionadded:: VERSION
Scrapy provides an option to activate plugins to post-process feeds before they are exported
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
can create your own :ref:`plugins <custom-plugins>`.
These plugins can be activated through the ``postprocessing`` option of a feed.
The option must be passed a list of post-processing plugins in the order you want
the feed to be processed. These plugins can be declared either as an import string
or with the imported class of the plugin. Parameters to plugins can be passed
through the feed options. See :ref:`feed options <feed-options>` for examples.
.. _builtin-plugins:
Built-in Plugins
----------------
.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
.. _custom-plugins:
Custom Plugins
--------------
Each plugin is a class that must implement the following methods:
.. method:: __init__(self, file, feed_options)
Initialize the plugin.
:param file: file-like object having at least the `write`, `tell` and `close` methods implemented
:param feed_options: feed-specific :ref:`options <feed-options>`
:type feed_options: :class:`dict`
.. method:: write(self, data)
Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
It must return number of bytes written.
.. method:: close(self)
Close the target file object.
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
can then access those parameters from the ``__init__`` method of your plugin.
Settings
@ -211,38 +375,131 @@ Settings
These are the settings used for configuring the feed exports:
* :setting:`FEED_URI` (mandatory)
* :setting:`FEED_FORMAT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
- :setting:`FEEDS` (mandatory)
- :setting:`FEED_EXPORT_ENCODING`
- :setting:`FEED_STORE_EMPTY`
- :setting:`FEED_EXPORT_FIELDS`
- :setting:`FEED_EXPORT_INDENT`
- :setting:`FEED_STORAGES`
- :setting:`FEED_STORAGE_FTP_ACTIVE`
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport
.. setting:: FEED_URI
.. setting:: FEEDS
FEED_URI
--------
FEEDS
-----
Default: ``None``
.. versionadded:: 2.1
The URI of the export feed. See :ref:`topics-feed-storage-backends` for
supported URI schemes.
Default: ``{}``
This setting is required for enabling the feed exports.
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
.. setting:: FEED_FORMAT
This setting is required for enabling the feed export feature.
FEED_FORMAT
-----------
See :ref:`topics-feed-storage-backends` for supported URI schemes.
The serialization format to be used for the feed. See
:ref:`topics-feed-format` for possible values.
For instance::
{
'items.json': {
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
'fields': None,
'indent': 4,
'item_export_kwargs': {
'export_empty_fields': True,
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],
'item_filter': MyCustomFilter1,
'encoding': 'latin1',
'indent': 8,
},
pathlib.Path('items.csv.gz'): {
'format': 'csv',
'fields': ['price', 'name'],
'item_filter': 'myproject.filters.MyCustomFilter2',
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
'gzip_compresslevel': 5,
},
}
.. _feed-options:
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition:
- ``format``: the :ref:`serialization format <topics-feed-format>`.
This setting is mandatory, there is no fallback value.
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
- ``item_classes``: list of :ref:`item classes <topics-items>` to export.
If undefined or empty, all items are exported.
.. versionadded:: VERSION
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
.. versionadded:: VERSION
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
.. versionadded:: 2.4.0
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
The default value depends on the :ref:`storage backend
<topics-feed-storage-backends>`:
- :ref:`topics-feed-storage-fs`: ``False``
- :ref:`topics-feed-storage-ftp`: ``True``
.. note:: Some FTP servers may not support appending to files (the
``APPE`` FTP command).
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
.. versionadded:: 2.4.0
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
- ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
The plugins will be used in the order of the list passed.
.. versionadded:: VERSION
.. setting:: FEED_EXPORT_ENCODING
@ -270,8 +527,8 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
defined in dicts or :class:`~.Item` subclasses a spider is yielding.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
defined in :ref:`item objects <topics-items>` yielded by your spider.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
@ -397,7 +654,117 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
'csv': None,
}
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
----------------------------
.. versionadded:: 2.3.0
Default: ``0``
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
storing up to the specified number of items in each output file.
When generating multiple output files, you must use at least one of the following
placeholders in the feed URI to indicate how the different output file names are
generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
alter the number format. For example, to make the batch ID a 5-digit
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
FEED_EXPORT_BATCH_ITEM_COUNT = 100
And your :command:`crawl` command line is::
scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
The command line above can generate a directory tree like::
->projectname
-->dirname
--->1-filename2020-03-28T14-45-08.237134.json
--->2-filename2020-03-28T14-45-09.148903.json
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
100 items or fewer.
.. setting:: FEED_URI_PARAMS
FEED_URI_PARAMS
---------------
Default: ``None``
A string with the import path of a function to set the parameters to apply with
:ref:`printf-style string formatting <python:old-string-formatting>` to the
feed URI.
The function signature should be as follows:
.. function:: uri_params(params, spider)
Return a :class:`dict` of key-value pairs to apply to the feed URI using
:ref:`printf-style string formatting <python:old-string-formatting>`.
:param params: default key-value pairs
Specifically:
- ``batch_id``: ID of the file batch. See
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
is always ``1``.
.. versionadded:: 2.3.0
- ``batch_time``: UTC date and time, in ISO format with ``:``
replaced with ``-``.
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``time``: ``batch_time``, with microseconds set to ``0``.
:type params: dict
:param spider: source spider of the feed items
:type spider: scrapy.Spider
For example, to include the :attr:`name <scrapy.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/

97
docs/topics/item-pipeline.rst
View File

@ -27,18 +27,22 @@ Each item pipeline component is a Python class that must implement the following
.. method:: process_item(self, item, spider)
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
(or any descendant class) object, return a
:class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
This method is called for every item pipeline component.
:param item: the item scraped
:type item: :class:`~scrapy.item.Item` object or a dict
`item` is an :ref:`item object <item-types>`, see
:ref:`supporting-item-types`.
:meth:`process_item` must either: return an :ref:`item object <item-types>`,
return a :class:`~twisted.internet.defer.Deferred` or raise a
:exc:`~scrapy.exceptions.DropItem` exception.
Dropped items are no longer processed by further pipeline components.
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
Additionally, they may also implement the following methods:
@ -47,14 +51,14 @@ Additionally, they may also implement the following methods:
This method is called when the spider is opened.
:param spider: the spider which was opened
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: close_spider(self, spider)
This method is called when the spider is closed.
:param spider: the spider which was closed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
@ -79,19 +83,20 @@ Let's take a look at the following hypothetical pipeline that adjusts the
(``price_excludes_vat`` attribute), and drops those items which don't
contain a price::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class PricePipeline(object):
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
if item.get('price'):
if item.get('price_excludes_vat'):
item['price'] = item['price'] * self.vat_factor
adapter = ItemAdapter(item)
if adapter.get('price'):
if adapter.get('price_excludes_vat'):
adapter['price'] = adapter['price'] * self.vat_factor
return item
else:
raise DropItem("Missing price in %s" % item)
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
@ -103,7 +108,9 @@ format::
import json
class JsonWriterPipeline(object):
from itemadapter import ItemAdapter
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'w')
@ -112,7 +119,7 @@ format::
self.file.close()
def process_item(self, item, spider):
line = json.dumps(dict(item)) + "\n"
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line)
return item
@ -131,8 +138,9 @@ The main point of this example is to show how to use :meth:`from_crawler`
method and how to clean up the resources properly.::
import pymongo
from itemadapter import ItemAdapter
class MongoPipeline(object):
class MongoPipeline:
collection_name = 'scrapy_items'
@ -155,57 +163,62 @@ method and how to clean up the resources properly.::
self.client.close()
def process_item(self, item, spider):
self.db[self.collection_name].insert_one(dict(item))
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item
.. _MongoDB: https://www.mongodb.com/
.. _pymongo: https://api.mongodb.com/python/current/
.. _ScreenshotPipeline:
Take screenshot of item
-----------------------
This example demonstrates how to return a
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded
and Deferred callback fires, it saves item to a file and adds filename to an item.
This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in
the :meth:`process_item` method.
This item pipeline makes a request to a locally-running instance of Splash_ to
render a screenshot of the item URL. After the request response is downloaded,
the item pipeline saves the screenshot to a file and adds the filename to the
item.
::
import scrapy
import hashlib
from urllib.parse import quote
import scrapy
from itemadapter import ItemAdapter
from scrapy.utils.defer import maybe_deferred_to_future
class ScreenshotPipeline(object):
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
every Scrapy item."""
SPLASH_URL = "http://localhost:8050/render.png?url={}"
def process_item(self, item, spider):
encoded_item_url = quote(item["url"])
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
dfd = spider.crawler.engine.download(request, spider)
dfd.addBoth(self.return_item, item)
return dfd
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
def return_item(self, response, item):
if response.status != 200:
# Error happened, return item.
return item
# Save screenshot to file, filename will be hash of url.
url = item["url"]
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
# Store filename in item.
item["screenshot_filename"] = filename
adapter["screenshot_filename"] = filename
return item
.. _Splash: https://splash.readthedocs.io/en/stable/
@ -218,18 +231,20 @@ already processed. Let's say that our items have a unique id, but our spider
returns multiples items with the same id::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class DuplicatesPipeline(object):
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()
def process_item(self, item, spider):
if item['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %s" % item)
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(item['id'])
self.ids_seen.add(adapter['id'])
return item

270
docs/topics/items.rst
View File

@ -8,31 +8,156 @@ Items
:synopsis: Item and Field classes
The main goal in scraping is to extract structured data from unstructured
sources, typically, web pages. Scrapy spiders can return the extracted data
as Python dicts. While convenient and familiar, Python dicts lack structure:
it is easy to make a typo in a field name or return inconsistent data,
especially in a larger project with many spiders.
sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the
extracted data as `items`, Python objects that define key-value pairs.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
their available fields.
Scrapy supports :ref:`multiple types of items <item-types>`. When you create an
item, you may use whichever type of item you want. When you write code that
receives an item, your code should :ref:`work for any item type
<supporting-item-types>`.
Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _item-types:
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
Item Types
==========
Scrapy supports the following types of items, via the `itemadapter`_ library:
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
.. _itemadapter: https://github.com/scrapy/itemadapter
.. _dict-items:
Dictionaries
------------
As an item type, :class:`dict` is convenient and familiar.
.. _item-objects:
Item objects
------------
:class:`Item` provides a :class:`dict`-like API plus additional features that
make it the most feature-complete item type:
.. class:: scrapy.item.Item([arg])
.. class:: scrapy.Item([arg])
:class:`Item` objects replicate the standard :class:`dict` API, including
its ``__init__`` method.
:class:`Item` allows defining field names, so that:
- :class:`KeyError` is raised when using undefined field names (i.e.
prevents typos going unnoticed)
- :ref:`Item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all
of them
:class:`Item` also allows defining field metadata, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
(see :ref:`topics-leaks-trackrefs`).
:class:`Item` objects also provide the following additional API members:
.. automethod:: copy
.. automethod:: deepcopy
.. attribute:: fields
A dictionary containing *all declared fields* for this Item, not only
those populated. The keys are the field names and the values are the
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Example::
from scrapy.item import Item, Field
class CustomItem(Item):
one_field = Field()
another_field = Field()
.. _dataclass-items:
Dataclass objects
-----------------
.. versionadded:: 2.2
:func:`~dataclasses.dataclass` allows defining item classes with field names,
so that :ref:`item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all of them.
Additionally, ``dataclass`` items also allow to:
* define the type and default value of each defined field.
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
They work natively in Python 3.7 or later, or using the `dataclasses
backport`_ in Python 3.6.
.. _dataclasses backport: https://pypi.org/project/dataclasses/
Example::
from dataclasses import dataclass
@dataclass
class CustomItem:
one_field: str
another_field: int
.. note:: Field types are not enforced at run time.
.. _attrs-items:
attr.s objects
--------------
.. versionadded:: 2.2
:func:`attr.s` allows defining item classes with field names,
so that :ref:`item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all of them.
Additionally, ``attr.s`` items also allow to:
* define the type and default value of each defined field.
* define custom field :ref:`metadata <attrs:metadata>`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
Example::
import attr
@attr.s
class CustomItem:
one_field = attr.ib()
another_field = attr.ib()
Working with Item objects
=========================
.. _topics-items-declaring:
Declaring Items
===============
Declaring Item subclasses
-------------------------
Items are declared using a simple class definition syntax and :class:`Field`
objects. Here is an example::
Item subclasses are declared using a simple class definition syntax and
:class:`Field` objects. Here is an example::
import scrapy
@ -50,10 +175,11 @@ objects. Here is an example::
.. _Django: https://www.djangoproject.com/
.. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
.. _topics-items-fields:
Item Fields
===========
Declaring fields
----------------
:class:`Field` objects are used to specify metadata for each field. For
example, the serializer function for the ``last_updated`` field illustrated in
@ -74,15 +200,32 @@ It's important to note that the :class:`Field` objects used to declare the item
do not stay assigned as class attributes. Instead, they can be accessed through
the :attr:`Item.fields` attribute.
Working with Items
==================
.. class:: scrapy.item.Field([arg])
.. class:: scrapy.Field([arg])
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs``
items. Please refer to the documentation for `dataclasses.field`_ and
`attr.ib`_ for additional information.
.. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field
.. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib
Working with Item objects
-------------------------
Here are some examples of common tasks performed with items, using the
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
notice the API is very similar to the `dict API`_.
notice the API is very similar to the :class:`dict` API.
Creating items
--------------
''''''''''''''
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
@ -90,7 +233,7 @@ Product(name='Desktop PC', price=1000)
Getting field values
--------------------
''''''''''''''''''''
>>> product['name']
Desktop PC
@ -130,7 +273,7 @@ False
Setting field values
--------------------
''''''''''''''''''''
>>> product['last_updated'] = 'today'
>>> product['last_updated']
@ -143,9 +286,9 @@ KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
''''''''''''''''''''''''''''''
To access all populated values, just use the typical `dict API`_:
To access all populated values, just use the typical :class:`dict` API:
>>> product.keys()
['price', 'name']
@ -157,16 +300,14 @@ To access all populated values, just use the typical `dict API`_:
.. _copying-items:
Copying items
-------------
'''''''''''''
To copy an item, you must first decide whether you want a shallow copy or a
deep copy.
If your item contains mutable_ values like lists or dictionaries, a shallow
copy will keep references to the same mutable values across all different
copies.
.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
If your item contains :term:`mutable` values like lists or dictionaries,
a shallow copy will keep references to the same mutable values across all
different copies.
For example, if you have an item with a list of tags, and you create a shallow
copy of that item, both the original item and the copy have the same list of
@ -175,21 +316,19 @@ other item as well.
If that is not the desired behavior, use a deep copy instead.
See the `documentation of the copy module`_ for more information.
.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
See :mod:`copy` for more information.
To create a shallow copy of an item, you can either call
:meth:`~scrapy.item.Item.copy` on an existing item
:meth:`~scrapy.Item.copy` on an existing item
(``product2 = product.copy()``) or instantiate your item class from an existing
item (``product2 = Product(product)``).
To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead
(``product2 = product.deepcopy()``).
Other common tasks
------------------
''''''''''''''''''
Creating dicts from items:
@ -207,8 +346,8 @@ Traceback (most recent call last):
KeyError: 'Product does not support field: lala'
Extending Items
===============
Extending Item subclasses
-------------------------
You can extend Items (to add more fields or to change some metadata for some
fields) by declaring a subclass of your original Item.
@ -228,46 +367,25 @@ appending more values, or changing existing values, like this::
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
keeping all the previously existing metadata values.
Item objects
============
.. class:: Item([arg])
.. _supporting-item-types:
Return a new Item optionally initialized from the given argument.
Supporting All Item Types
=========================
Items replicate the standard `dict API`_, including its ``__init__`` method, and
also provide the following additional API members:
In code that receives an item, such as methods of :ref:`item pipelines
<topics-item-pipeline>` or :ref:`spider middlewares
<topics-spider-middleware>`, it is a good practice to use the
:class:`~itemadapter.ItemAdapter` class and the
:func:`~itemadapter.is_item` function to write code that works for
any :ref:`supported item type <item-types>`:
.. automethod:: copy
.. autoclass:: itemadapter.ItemAdapter
.. automethod:: deepcopy
.. attribute:: fields
A dictionary containing *all declared fields* for this Item, not only
those populated. The keys are the field names and the values are the
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
Field objects
=============
.. class:: Field([arg])
The :class:`Field` class is just an alias to the built-in `dict`_ class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
.. autofunction:: itemadapter.is_item
Other classes related to Item
=============================
.. autoclass:: BaseItem
Other classes related to items
==============================
.. autoclass:: ItemMeta

11
docs/topics/jobs.rst
View File

@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
.. _topics-keeping-persistent-state-between-batches:
Keeping persistent state between batches
========================================
@ -65,16 +67,19 @@ Cookies expiration
------------------
Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
scheduled may no longer work. This won't be an issue if your spider doesn't rely
on cookies.
.. _request-serialization:
Request serialization
---------------------
For persistence to work, :class:`~scrapy.http.Request` objects must be
For persistence to work, :class:`~scrapy.Request` objects must be
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
values passed to their ``__init__`` method, which must be methods of the
running :class:`~scrapy.spiders.Spider` class.
running :class:`~scrapy.Spider` class.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.

103
docs/topics/leaks.rst
View File

@ -4,7 +4,7 @@
Debugging memory leaks
======================
In Scrapy, objects such as Requests, Responses and Items have a finite
In Scrapy, objects such as requests, responses and items have a finite
lifetime: they are created, used for a while, and finally destroyed.
From all those objects, the Request is probably the one with the longest
@ -17,8 +17,8 @@ what is known as a "memory leak".
To help debugging memory leaks, Scrapy provides a built-in mechanism for
tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
and you can also use a third-party library called :ref:`Guppy
<topics-leaks-guppy>` for more advanced memory debugging (see below for more
and you can also use a third-party library called :ref:`muppy
<topics-leaks-muppy>` for more advanced memory debugging (see below for more
info). Both mechanisms must be used from the :ref:`Telnet Console
<topics-telnetconsole>`.
@ -27,7 +27,7 @@ Common causes of memory leaks
It happens quite often (sometimes by accident, sometimes on purpose) that the
Scrapy developer passes objects referenced in Requests (for example, using the
:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta`
attributes or the request callback function) and that effectively bounds the
lifetime of those referenced objects to the lifetime of the Request. This is,
by far, the most common cause of memory leaks in Scrapy projects, and a quite
@ -48,9 +48,9 @@ Too Many Requests?
------------------
By default Scrapy keeps the request queue in memory; it includes
:class:`~scrapy.http.Request` objects and all objects
referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
and :attr:`~scrapy.http.Request.meta`).
:class:`~scrapy.Request` objects and all objects
referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs`
and :attr:`~scrapy.Request.meta`).
While not necessarily a leak, this can take a lot of memory. Enabling
:ref:`persistent job queue <topics-jobs>` could help keeping memory usage
in control.
@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref``
========================================
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
memory leaks. It basically tracks the references to all live Requests,
Responses, Item and Selector objects.
memory leaks. It basically tracks the references to all live Request,
Response, Item, Spider and Selector objects.
You can enter the telnet console and inspect how many objects (of the classes
mentioned above) are currently alive using the ``prefs()`` function which is an
@ -90,11 +90,11 @@ Which objects are tracked?
The objects tracked by ``trackrefs`` are all from these classes (and all its
subclasses):
* :class:`scrapy.http.Request`
* :class:`scrapy.Request`
* :class:`scrapy.http.Response`
* :class:`scrapy.item.Item`
* :class:`scrapy.selector.Selector`
* :class:`scrapy.spiders.Spider`
* :class:`scrapy.Item`
* :class:`scrapy.Selector`
* :class:`scrapy.Spider`
A real example
--------------
@ -102,7 +102,7 @@ A real example
Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
.. class:: object_ref
Inherit from this class (instead of object) if you want to track live
Inherit from this class if you want to track live
instances with the ``trackref`` module.
.. function:: print_live_refs(class_name, ignore=NoneType)
@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
:param ignore: if given, all objects from the specified class (or tuple of
classes) will be ignored.
:type ignore: class or classes tuple
:type ignore: type or tuple
.. function:: get_oldest(class_name)
@ -193,73 +193,18 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
``None`` if none is found. Use :func:`print_live_refs` first to get a list
of all tracked live objects per class name.
.. _topics-leaks-guppy:
Debugging memory leaks with Guppy
=================================
``trackref`` provides a very convenient mechanism for tracking down memory
leaks, but it only keeps track of the objects that are more likely to cause
memory leaks (Requests, Responses, Items, and Selectors). However, there are
other cases where the memory leaks could come from other (more or less obscure)
objects. If this is your case, and you can't find your leaks using ``trackref``,
you still have another resource: the `Guppy library`_.
If you're using Python3, see :ref:`topics-leaks-muppy`.
.. _Guppy library: https://pypi.org/project/guppy/
If you use ``pip``, you can install Guppy with the following command::
pip install guppy
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
Guppy heap objects. Here's an example to view all Python objects available in
the heap using Guppy:
>>> x = hpy.heap()
>>> x.bytype
Partition of a set of 297033 objects. Total size = 52587824 bytes.
Index Count % Size % Cumulative % Type
0 22307 8 16423880 31 16423880 31 dict
1 122285 41 12441544 24 28865424 55 str
2 68346 23 5966696 11 34832120 66 tuple
3 227 0 5836528 11 40668648 77 unicode
4 2461 1 2222272 4 42890920 82 type
5 16870 6 2024400 4 44915320 85 function
6 13949 5 1673880 3 46589200 89 types.CodeType
7 13422 5 1653104 3 48242304 92 list
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
<1676 more rows. Type e.g. '_.more' to view.>
You can see that most space is used by dicts. Then, if you want to see from
which attribute those dicts are referenced, you could do:
>>> x.bytype[0].byvia
Partition of a set of 22307 objects. Total size = 16423880 bytes.
Index Count % Size % Cumulative % Referred Via:
0 10982 49 9416336 57 9416336 57 '.__dict__'
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
2 3097 14 1122904 7 13220744 80
3 990 4 277200 2 13497944 82 "['cookies']"
4 987 4 276360 2 13774304 84 "['cache']"
5 985 4 275800 2 14050104 86 "['meta']"
6 897 4 251160 2 14301264 87 '[2]'
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
9 27 0 155016 1 14841328 90 '[1]'
<333 more rows. Type e.g. '_.more' to view.>
As you can see, the Guppy module is very powerful but also requires some deep
knowledge about Python internals. For more info about Guppy, refer to the
`Guppy documentation`_.
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
.. _topics-leaks-muppy:
Debugging memory leaks with muppy
=================================
``trackref`` provides a very convenient mechanism for tracking down memory
leaks, but it only keeps track of the objects that are more likely to cause
memory leaks. However, there are other cases where the memory leaks could come
from other (more or less obscure) objects. If this is your case, and you can't
find your leaks using ``trackref``, you still have another resource: the muppy
library.
You can use muppy from `Pympler`_.
.. _Pympler: https://pypi.org/project/Pympler/

45
docs/topics/link-extractors.rst
View File

@ -10,12 +10,19 @@ The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
list of matching :class:`~scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
def parse(self, response):
for link in self.link_extractor.extract_links(response):
yield Request(link.url, callback=self.parse)
.. _topics-link-extractors-ref:
@ -46,13 +53,13 @@ LxmlLinkExtractor
:param allow: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be extracted. If not
given (or empty), it will match all links.
:type allow: a regular expression (or list of)
:type allow: str or list
:param deny: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be excluded (i.e. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type deny: a regular expression (or list of)
:type deny: str or list
:param allow_domains: a single value or a list of string containing
domains which will be considered for extracting the links
@ -64,9 +71,13 @@ LxmlLinkExtractor
:param deny_extensions: a single value or list of strings containing
extensions that should be ignored when extracting links.
If not given, it will default to the
``IGNORED_EXTENSIONS`` list defined in the
`scrapy.linkextractors`_ package.
If not given, it will default to
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
.. versionchanged:: 2.0
:data:`~scrapy.linkextractors.IGNORED_EXTENSIONS` now includes
``7z``, ``7zip``, ``apk``, ``bz2``, ``cdr``, ``dmg``, ``ico``,
``iso``, ``tar``, ``tar.gz``, ``webm``, and ``xz``.
:type deny_extensions: list
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
@ -84,7 +95,7 @@ LxmlLinkExtractor
that the link's text must match in order to be extracted. If not
given (or empty), it will match all links. If a list of regular expressions is
given, the link will be extracted if it matches at least one.
:type restrict_text: a regular expression (or list of)
:type restrict_text: str or list
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
@ -102,11 +113,11 @@ LxmlLinkExtractor
different for requests with canonicalized and raw URLs. If you're
using LinkExtractor to follow links it is more robust to
keep the default ``canonicalize=False``.
:type canonicalize: boolean
:type canonicalize: bool
:param unique: whether duplicate filtering should be applied to extracted
links.
:type unique: boolean
:type unique: bool
:param process_value: a function which receives each value extracted from
the tag and attributes scanned and can modify the value and return a
@ -128,7 +139,7 @@ LxmlLinkExtractor
if m:
return m.group(1)
:type process_value: callable
:type process_value: collections.abc.Callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
@ -137,8 +148,16 @@ LxmlLinkExtractor
elements, etc., so LinkExtractor strips space chars by default.
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
:type strip: bool
.. automethod:: extract_links
Link
----
.. module:: scrapy.link
:synopsis: Link from link extractors
.. autoclass:: Link
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

481
docs/topics/loaders.rst
View File

@ -7,13 +7,12 @@ Item Loaders
.. module:: scrapy.loader
:synopsis: Item Loader class
Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
<topics-items>`. Even though Items can be populated using their own
dictionary-like API, Item Loaders provide a much more convenient API for
populating them from a scraping process, by automating some common tasks like
parsing the raw extracted data before assigning it.
Item Loaders provide a convenient mechanism for populating scraped :ref:`items
<topics-items>`. Even though items can be populated directly, Item Loaders provide a
much more convenient API for populating them from a scraping process, by automating
some common tasks like parsing the raw extracted data before assigning it.
In other words, :ref:`Items <topics-items>` provide the *container* of
In other words, :ref:`items <topics-items>` provide the *container* of
scraped data, while Item Loaders provide the mechanism for *populating* that
container.
@ -21,14 +20,18 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
for extending and overriding different field parsing rules, either by spider,
or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
.. note:: Item Loaders are an extension of the itemloaders_ library that make it
easier to work with Scrapy by adding support for
:ref:`responses <topics-request-response>`.
Using Item Loaders to populate items
====================================
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
instantiate it with an :ref:`item object <topics-items>` or without one, in which
case an :ref:`item object <topics-items>` is automatically created in the
Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
specified in the :attr:`ItemLoader.default_item_class` attribute.
Then, you start collecting values into the Item Loader, typically using
:ref:`Selectors <topics-selectors>`. You can add more than one value to
@ -53,7 +56,7 @@ chapter <topics-items>`::
l.add_xpath('name', '//div[@class="product_name"]')
l.add_xpath('name', '//div[@class="product_title"]')
l.add_xpath('price', '//p[@id="price"]')
l.add_css('stock', 'p#stock]')
l.add_css('stock', 'p#stock')
l.add_value('last_updated', 'today') # you can also use literal values
return l.load_item()
@ -77,6 +80,31 @@ called which actually returns the item populated with the data
previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
.. _topics-loaders-dataclass:
Working with dataclass items
============================
By default, :ref:`dataclass items <dataclass-items>` require all fields to be
passed when created. This could be an issue when using dataclass items with
item loaders: unless a pre-populated item is passed to the loader, fields
will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
One approach to overcome this is to define items using the
:func:`~dataclasses.field` function, with a ``default`` argument::
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: Optional[str] = field(default=None)
price: Optional[float] = field(default=None)
stock: Optional[int] = field(default=None)
.. _topics-loaders-processors:
Input and Output processors
@ -88,7 +116,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`
:meth:`~ItemLoader.add_value` methods) and the result of the input processor is
collected and kept inside the ItemLoader. After collecting all data, the
:meth:`ItemLoader.load_item` method is called to populate and get the populated
:class:`~scrapy.item.Item` object. That's when the output processor is
:ref:`item object <topics-items>`. That's when the output processor is
called with the data previously collected (and processed using the input
processor). The result of the output processor is the final value that gets
assigned to the item.
@ -136,6 +164,9 @@ with the data to be parsed, and return a parsed value. So you can use any
function as input or output processor. The only requirement is that they must
accept one (and only one) positional argument, which will be an iterable.
.. versionchanged:: 2.0
Processors no longer need to be methods.
.. note:: Both input and output processors must receive an iterable as their
first argument. The output of those functions can be anything. The result of
input processors will be appended to an internal list (in the Loader)
@ -146,28 +177,26 @@ The other thing you need to keep in mind is that the values returned by input
processors are collected internally (in lists) and then passed to output
processors to populate the fields.
Last, but not least, Scrapy comes with some :ref:`commonly used processors
<topics-loaders-available-processors>` built-in for convenience.
Last, but not least, itemloaders_ comes with some :ref:`commonly used
processors <itemloaders:built-in-processors>` built-in for convenience.
Declaring Item Loaders
======================
Item Loaders are declared like Items, by using a class definition syntax. Here
is an example::
Item Loaders are declared using a class definition syntax. Here is an example::
from itemloaders.processors import TakeFirst, MapCompose, Join
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst, MapCompose, Join
class ProductLoader(ItemLoader):
default_output_processor = TakeFirst()
name_in = MapCompose(unicode.title)
name_in = MapCompose(str.title)
name_out = Join()
price_in = MapCompose(unicode.strip)
price_in = MapCompose(str.strip)
# ...
@ -189,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
metadata. Here is an example::
import scrapy
from scrapy.loader.processors import Join, MapCompose, TakeFirst
from itemloaders.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_price(value):
@ -208,10 +237,10 @@ metadata. Here is an example::
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:
@ -270,248 +299,9 @@ There are several ways to modify Item Loader context values:
ItemLoader objects
==================
.. class:: ItemLoader([item, selector, response], \**kwargs)
Return a new Item Loader for populating the given Item. If no item is
given, one is instantiated automatically using the class in
:attr:`default_item_class`.
When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
data from web pages using :ref:`selectors <topics-selectors>`.
:param item: The item instance to populate using subsequent calls to
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
or :meth:`~ItemLoader.add_value`.
:type item: :class:`~scrapy.item.Item` object
:param selector: The selector to extract data from, when using the
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
(resp. :meth:`replace_css`) method.
:type selector: :class:`~scrapy.selector.Selector` object
:param response: The response used to construct the selector using the
:attr:`default_selector_class`, unless the selector argument is given,
in which case this argument is ignored.
:type response: :class:`~scrapy.http.Response` object
The item, selector, response and the remaining keyword arguments are
assigned to the Loader context (accessible through the :attr:`context` attribute).
:class:`ItemLoader` instances have the following methods:
.. method:: get_value(value, \*processors, \**kwargs)
Process the given ``value`` by the given ``processors`` and keyword
arguments.
Available keyword arguments:
:param re: a regular expression to use for extracting data from the
given value using :meth:`~scrapy.utils.misc.extract_regex` method,
applied before processors
:type re: str or compiled regex
Examples:
>>> from scrapy.loader.processors import TakeFirst
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
.. method:: add_value(field_name, value, \*processors, \**kwargs)
Process and then add the given ``value`` for the given field.
The value is first passed through :meth:`get_value` by giving the
``processors`` and ``kwargs``, and then passed through the
:ref:`field input processor <topics-loaders-processors>` and its result
appended to the data collected for that field. If the field already
contains collected data, the new data is added.
The given ``field_name`` can be ``None``, in which case values for
multiple fields may be added. And the processed value should be a dict
with field_name mapped to values.
Examples::
loader.add_value('name', u'Color TV')
loader.add_value('colours', [u'white', u'blue'])
loader.add_value('length', u'100')
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
.. method:: replace_value(field_name, value, \*processors, \**kwargs)
Similar to :meth:`add_value` but replaces the collected data with the
new value instead of adding it.
.. method:: get_xpath(xpath, \*processors, \**kwargs)
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
selector associated with this :class:`ItemLoader`.
:param xpath: the XPath to extract data from
:type xpath: str
:param re: a regular expression to use for extracting data from the
selected XPath region
:type re: str or compiled regex
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.get_xpath('//p[@class="product-name"]')
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
.. method:: add_xpath(field_name, xpath, \*processors, \**kwargs)
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
selector associated with this :class:`ItemLoader`.
See :meth:`get_xpath` for ``kwargs``.
:param xpath: the XPath to extract data from
:type xpath: str
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.add_xpath('name', '//p[@class="product-name"]')
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
.. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs)
Similar to :meth:`add_xpath` but replaces collected data instead of
adding it.
.. method:: get_css(css, \*processors, \**kwargs)
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
from the selector associated with this :class:`ItemLoader`.
:param css: the CSS selector to extract data from
:type css: str
:param re: a regular expression to use for extracting data from the
selected CSS region
:type re: str or compiled regex
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.get_css('p.product-name')
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
.. method:: add_css(field_name, css, \*processors, \**kwargs)
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
from the selector associated with this :class:`ItemLoader`.
See :meth:`get_css` for ``kwargs``.
:param css: the CSS selector to extract data from
:type css: str
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.add_css('name', 'p.product-name')
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_css('price', 'p#price', re='the price is (.*)')
.. method:: replace_css(field_name, css, \*processors, \**kwargs)
Similar to :meth:`add_css` but replaces collected data instead of
adding it.
.. method:: load_item()
Populate the item with the data collected so far, and return it. The
data collected is first passed through the :ref:`output processors
<topics-loaders-processors>` to get the final value to assign to each
item field.
.. method:: nested_xpath(xpath)
Create a nested loader with an xpath selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
.. method:: nested_css(css)
Create a nested loader with a css selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
.. method:: get_collected_values(field_name)
Return the collected values for the given field.
.. method:: get_output_value(field_name)
Return the collected values parsed using the output processor, for the
given field. This method doesn't populate or modify the item at all.
.. method:: get_input_processor(field_name)
Return the input processor for the given field.
.. method:: get_output_processor(field_name)
Return the output processor for the given field.
:class:`ItemLoader` instances have the following attributes:
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
This is mostly used as a property so when attempting to override this
value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
The currently active :ref:`Context <topics-loaders-context>` of this
Item Loader.
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
the ``__init__`` method.
.. attribute:: default_input_processor
The default input processor to use for those fields which don't specify
one.
.. attribute:: default_output_processor
The default output processor to use for those fields which don't specify
one.
.. attribute:: default_selector_class
The class used to construct the :attr:`selector` of this
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
If a selector is given in the ``__init__`` method this attribute is ignored.
This attribute is sometimes overridden in subclasses.
.. attribute:: selector
The :class:`~scrapy.selector.Selector` object to extract data from.
It's either the selector given in the ``__init__`` method or one created from
the response given in the ``__init__`` method using the
:attr:`default_selector_class`. This attribute is meant to be
read-only.
.. autoclass:: scrapy.loader.ItemLoader
:members:
:inherited-members:
.. _topics-loaders-nested:
@ -582,7 +372,7 @@ those dashes in the final product names.
Here's how you can remove those dashes by reusing and extending the default
Product Item Loader (``ProductLoader``)::
from scrapy.loader.processors import MapCompose
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
def strip_dashes(x):
@ -595,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
multiple source formats, for example XML and HTML. In the XML version you may
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
from scrapy.loader.processors import MapCompose
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
from myproject.utils.xml import remove_cdata
@ -615,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
organization of your Loaders collection - that's up to you and your project's
needs.
.. _topics-loaders-available-processors:
Available built-in processors
=============================
.. module:: scrapy.loader.processors
:synopsis: A collection of processors to use with Item Loaders
Even though you can use any callable function as input and output processors,
Scrapy provides some commonly used processors, which are described below. Some
of them, like the :class:`MapCompose` (which is typically used as input
processor) compose the output of several functions executed in order, to
produce the final parsed value.
Here is a list of all built-in processors:
.. class:: Identity
The simplest processor, which doesn't do anything. It returns the original
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
Example:
>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['one', 'two', 'three'])
['one', 'two', 'three']
.. class:: TakeFirst
Returns the first non-null/non-empty value from the values received,
so it's typically used as an output processor to single-valued fields.
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
Example:
>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'one', 'two', 'three'])
'one'
.. class:: Join(separator=u' ')
Returns the values joined with the separator given in the ``__init__`` method, which
defaults to ``u' '``. It doesn't accept Loader contexts.
When using the default separator, this processor is equivalent to the
function: ``u' '.join``
Examples:
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
'one two three'
>>> proc = Join('<br>')
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
.. class:: Compose(\*functions, \**default_loader_context)
A processor which is constructed from the composition of the given
functions. This means that each input value of this processor is passed to
the first function, and the result of that function is passed to the second
function, and so on, until the last function returns the output value of
this processor.
By default, stop process on ``None`` value. This behaviour can be changed by
passing keyword argument ``stop_on_none=False``.
Example:
>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['hello', 'world'])
'HELLO'
Each function can optionally receive a ``loader_context`` parameter. For
those which do, this processor will pass the currently active :ref:`Loader
context <topics-loaders-context>` through that parameter.
The keyword arguments passed in the ``__init__`` method are used as the default
Loader context values passed to each function call. However, the final
Loader context values passed to functions are overridden with the currently
active Loader context accessible through the :meth:`ItemLoader.context`
attribute.
.. class:: MapCompose(\*functions, \**default_loader_context)
A processor which is constructed from the composition of the given
functions, similar to the :class:`Compose` processor. The difference with
this processor is the way internal results are passed among functions,
which is as follows:
The input value of this processor is *iterated* and the first function is
applied to each element. The results of these function calls (one for each element)
are concatenated to construct a new iterable, which is then used to apply the
second function, and so on, until the last function is applied to each
value of the list of values collected so far. The output values of the last
function are concatenated together to produce the output of this processor.
Each particular function can return a value or a list of values, which is
flattened with the list of values returned by the same function applied to
the other input values. The functions can also return ``None`` in which
case the output of that function is ignored for further processing over the
chain.
This processor provides a convenient way to compose functions that only
work with single values (instead of iterables). For this reason the
:class:`MapCompose` processor is typically used as input processor, since
data is often extracted using the
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
<topics-selectors>`, which returns a list of unicode strings.
The example below should clarify how it works:
>>> def filter_world(x):
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
>>> proc = MapCompose(filter_world, str.upper)
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
``__init__`` method keyword arguments are used as default context values. See
:class:`Compose` processor for more info.
.. class:: SelectJmes(json_path)
Queries the value using the json path provided to the ``__init__`` method and returns the output.
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
Example:
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
>>> proc({'foo': 'bar'})
'bar'
>>> proc({'foo': {'bar': 'baz'}})
{'bar': 'baz'}
Working with Json:
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
['bar']
.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html

83
docs/topics/logging.rst
View File

@ -9,8 +9,7 @@ Logging
explicit calls to the Python standard logging. Keep reading to learn more
about the new logging system.
Scrapy uses `Python's builtin logging system
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
Scrapy uses :mod:`logging` for event logging. We'll
provide some simple examples to get you started, but for more advanced
use-cases it's strongly suggested to read thoroughly its documentation.
@ -83,10 +82,10 @@ path::
.. seealso::
Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
Module logging, :doc:`HowTo <howto/logging>`
Basic Logging Tutorial
Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
Module logging, :ref:`Loggers <logger>`
Further documentation on loggers
.. _topics-logging-from-spiders:
@ -94,7 +93,7 @@ path::
Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
instance, which can be accessed and used like this::
import scrapy
@ -102,7 +101,7 @@ instance, which can be accessed and used like this::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapinghub.com']
start_urls = ['https://scrapy.org']
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
@ -118,7 +117,7 @@ Python logger you want. For example::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapinghub.com']
start_urls = ['https://scrapy.org']
def parse(self, response):
logger.info('Parse function called on %s', response.url)
@ -144,6 +143,7 @@ Logging settings
These settings can be used to configure the logging:
* :setting:`LOG_FILE`
* :setting:`LOG_FILE_APPEND`
* :setting:`LOG_ENABLED`
* :setting:`LOG_ENCODING`
* :setting:`LOG_LEVEL`
@ -156,7 +156,9 @@ The first couple of settings define a destination for log messages. If
:setting:`LOG_FILE` is set, messages sent through the root logger will be
redirected to a file named :setting:`LOG_FILE` with encoding
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
messages will be displayed on the standard error. Lastly, if
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
(discarding the output from previous runs, if any). Lastly, if
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
@ -165,14 +167,12 @@ possible levels listed in :ref:`topics-logging-levels`.
:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
used as layouts for all messages. Those strings can contain any placeholders
listed in `logging's logrecord attributes docs
<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
`datetime's strftime and strptime directives
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
respectively.
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
component that prints the log. It is unset by default, hence logs contain the
Scrapy component responsible for that log output.
Command-line options
@ -190,7 +190,7 @@ to override some of the Scrapy settings regarding logging.
.. seealso::
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
Module :mod:`logging.handlers`
Further documentation on available handlers
.. _custom-log-formats:
@ -201,10 +201,13 @@ Custom Log Formats
A custom log format can be set for different actions by extending
:class:`~scrapy.logformatter.LogFormatter` class and making
:setting:`LOG_FORMATTER` point to your new class.
.. autoclass:: scrapy.logformatter.LogFormatter
:members:
.. _topics-logging-advanced-customization:
Advanced customization
----------------------
@ -242,6 +245,47 @@ e.g. in the spider's ``__init__`` method::
If you run this spider again then INFO messages from
``scrapy.spidermiddlewares.httperror`` logger will be gone.
You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================
@ -256,16 +300,15 @@ scrapy.utils.log module
In that case, its usage is not required but it's recommended.
Another option when running custom scripts is to manually configure the logging.
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
To do this you can use :func:`logging.basicConfig` to set a basic root handler.
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
so it is recommended to only use `logging.basicConfig()`_ together with
so it is recommended to only use :func:`logging.basicConfig` together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
import logging
from scrapy.utils.log import configure_logging
logging.basicConfig(
filename='log.txt',
@ -275,7 +318,3 @@ scrapy.utils.log module
Refer to :ref:`run-from-script` for more details about using Scrapy this
way.
.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig

202
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:
@ -50,12 +50,14 @@ this:
4. When the files are downloaded, another field (``files``) will be populated
with the results. This field will contain a list of dicts with information
about the downloaded files, such as the downloaded path, the original
scraped url (taken from the ``file_urls`` field) , and the file checksum.
scraped url (taken from the ``file_urls`` field), the file checksum and the file status.
The files in the list of the ``files`` field will retain the same order of
the original ``file_urls`` field. If some file failed downloading, an
error will be logged and the file won't be present in the ``files`` field.
.. _images-pipeline:
Using the Images Pipeline
=========================
@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
JPEG/RGB format, so you need to install this library in order to use it.
`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
PIL.
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
.. _Python Imaging Library: http://www.pythonware.com/products/pil/
.. _topics-media-pipeline-enabling:
@ -113,31 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
IMAGES_STORE = '/path/to/valid/dir'
Supported Storage
=================
.. _topics-file-naming:
File system is currently the only officially supported storage, but there are
also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
File Naming
===========
.. _Amazon S3: https://aws.amazon.com/s3/
.. _Google Cloud Storage: https://cloud.google.com/storage/
File system storage
Default File Naming
-------------------
The files are stored using a `SHA1 hash`_ of their URLs for the file names.
By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names.
For example, the following image URL::
http://www.example.com/image.jpg
Whose ``SHA1 hash`` is::
Whose ``SHA-1 hash`` is::
3afec3b4765f8f0a07b78f98c07b83f013567a0a
Will be downloaded and stored in the following file::
Will be downloaded and stored using your chosen :ref:`storage method <topics-supported-storage>` and the following file name::
<IMAGES_STORE>/full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
Custom File Naming
-------------------
You may wish to use a different calculated file name for saved files.
For example, classifying an image by including meta in the file name.
Customize file names by overriding the ``file_path`` method of your
media pipeline.
For example, an image pipeline with image URL::
http://www.example.com/product/images/large/front/0000000004166
Can be processed into a file name with a condensed hash and the perspective
``front``::
00b08510e4_front.jpg
By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
image_perspective = request.url.split('/')[-2]
image_filename = f'{image_url_hash}_{image_perspective}.jpg'
return image_filename
.. warning::
If your custom file name scheme relies on meta data that can vary between
scrapes it may lead to unexpected re-downloading of existing media using
new file names.
For example, if your custom file name scheme uses a product title and the
site changes an item's product title between scrapes, Scrapy will re-download
the same media using updated file names.
For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`.
.. _topics-supported-storage:
Supported Storage
=================
File system storage
-------------------
File system storage will save files to the following path::
<IMAGES_STORE>/full/<FILE_NAME>
Where:
@ -147,9 +196,16 @@ Where:
* ``full`` is a sub-directory to separate full images from thumbnails (if
used). For more info see :ref:`topics-images-thumbnails`.
* ``<FILE_NAME>`` is the file name assigned to the file. For more info see :ref:`topics-file-naming`.
.. _media-pipeline-ftp:
FTP server storage
------------------
.. versionadded:: 2.0
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
Scrapy will automatically upload the files to the server.
@ -158,7 +214,7 @@ following forms::
ftp://username:password@address:port/path
ftp://address:port/path
If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
:setting:`FTP_PASSWORD` settings respectively.
@ -166,14 +222,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
the passive connection mode by default. To use the active connection mode instead,
set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
.. _media-pipelines-s3:
Amazon S3 storage
-----------------
.. setting:: FILES_STORE_S3_ACL
.. setting:: IMAGES_STORE_S3_ACL
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
bucket. Scrapy will automatically upload the files to the bucket.
If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
@ -189,8 +248,9 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@ -199,14 +259,17 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Minio: https://github.com/minio/minio
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _media-pipeline-gcs:
Google Cloud Storage
---------------------
.. setting:: GCS_PROJECT_ID
.. setting:: FILES_STORE_GCS_ACL
.. setting:: IMAGES_STORE_GCS_ACL
@ -245,20 +308,22 @@ Usage example
.. setting:: IMAGES_URLS_FIELD
.. setting:: IMAGES_RESULT_FIELD
In order to use a media pipeline first, :ref:`enable it
In order to use a media pipeline, first :ref:`enable it
<topics-media-pipeline-enabling>`.
Then, if a spider returns a dict with the URLs key (``file_urls`` or
``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
put the results under respective key (``files`` or ``images``).
Then, if a spider returns an :ref:`item object <topics-items>` with the URLs
field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline
respectively), the pipeline will put the results under the respective field
(``files`` or ``images``).
If you prefer to use :class:`~.Item`, then define a custom item with the
necessary fields, like in this example for Images Pipeline::
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.Item` class::
import scrapy
class MyItem(scrapy.Item):
# ... other item fields ...
image_urls = scrapy.Field()
images = scrapy.Field()
@ -318,6 +383,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
and pipeline class MyPipeline will have expiration time set to 180.
The last modified time from the file is used to determine the age of the file in days,
which is then compared to the set expiration time to determine if the file is expired.
.. _topics-images-thumbnails:
Thumbnail generation for images
@ -348,9 +416,9 @@ Where:
* ``<size_name>`` is the one specified in the :setting:`IMAGES_THUMBS`
dictionary keys (``small``, ``big``, etc)
* ``<image_id>`` is the `SHA1 hash`_ of the image url
* ``<image_id>`` is the `SHA-1 hash`_ of the image url
.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
Example of image files stored using ``small`` and ``big`` thumbnail names::
@ -410,15 +478,16 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.Item>`
You can override this method to customize the download path of each file.
@ -434,12 +503,18 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@ -447,8 +522,11 @@ See here the methods that you can override in your custom Files Pipeline:
:meth:`~get_media_requests` method and return a Request for each
file URL::
from itemadapter import ItemAdapter
def get_media_requests(self, item, info):
for file_url in item['file_urls']:
adapter = ItemAdapter(item)
for file_url in adapter['file_urls']:
yield scrapy.Request(file_url)
Those requests will be processed by the pipeline and, when they have finished
@ -472,6 +550,18 @@ See here the methods that you can override in your custom Files Pipeline:
* ``checksum`` - a `MD5 hash`_ of the image contents
* ``status`` - the file status indication.
.. versionadded:: 2.2
It can be one of the following:
* ``downloaded`` - file was downloaded.
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
according to the file expiration policy.
* ``cached`` - file was already scheduled for download, by another item
sharing the same file.
The list of tuples received by :meth:`~item_completed` is
guaranteed to retain the same order of the requests returned from the
:meth:`~get_media_requests` method.
@ -481,7 +571,8 @@ See here the methods that you can override in your custom Files Pipeline:
[(True,
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
'url': 'http://www.example.com/files/product1.pdf'}),
'url': 'http://www.example.com/files/product1.pdf',
'status': 'downloaded'}),
(False,
Failure(...))]
@ -502,13 +593,15 @@ See here the methods that you can override in your custom Files Pipeline:
store the downloaded file paths (passed in results) in the ``file_paths``
item field, and we drop the item if it doesn't contain any files::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
def item_completed(self, results, item, info):
file_paths = [x['path'] for ok, x in results if ok]
if not file_paths:
raise DropItem("Item contains no files")
item['file_paths'] = file_paths
adapter = ItemAdapter(item)
adapter['file_paths'] = file_paths
return item
By default, the :meth:`item_completed` method returns the item.
@ -524,15 +617,16 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.Item>`
You can override this method to customize the download path of each file.
@ -548,12 +642,18 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
@ -573,6 +673,8 @@ See here the methods that you can override in your custom Images Pipeline:
By default, the :meth:`item_completed` method returns the item.
.. _media-pipeline-example:
Custom Images pipeline example
==============================
@ -580,8 +682,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified
above::
import scrapy
from scrapy.pipelines.images import ImagesPipeline
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
@ -593,7 +696,8 @@ above::
image_paths = [x['path'] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
item['image_paths'] = image_paths
adapter = ItemAdapter(item)
adapter['image_paths'] = image_paths
return item

42
docs/topics/practices.rst
View File

@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it.
...
process = CrawlerProcess(settings={
'FEED_FORMAT': 'json',
'FEED_URI': 'items.json'
"FEEDS": {
"items.json": {"format": "json"},
},
})
process.crawl(MySpider)
@ -62,7 +63,7 @@ project as example.
process = CrawlerProcess(get_project_settings())
# 'followall' is the name of one of the spiders of the project.
process.crawl('followall', domain='scrapinghub.com')
process.crawl('followall', domain='scrapy.org')
process.start() # the script will block here until the crawling is finished
There's another Scrapy utility that provides more control over the crawling
@ -118,6 +119,7 @@ Here is an example that runs multiple spiders simultaneously:
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -127,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously:
# Your second spider definition
...
process = CrawlerProcess()
settings = get_project_settings()
process = CrawlerProcess(settings)
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start() # the script will block here until all crawling jobs are finished
@ -140,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -150,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
...
configure_logging()
runner = CrawlerRunner()
settings = get_project_settings()
runner = CrawlerRunner(settings)
runner.crawl(MySpider1)
runner.crawl(MySpider2)
d = runner.join()
@ -165,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -175,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
...
configure_logging()
runner = CrawlerRunner()
settings = get_project_settings()
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
def crawl():
@ -186,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
crawl()
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some
limitations, to use these different values. What happens in practice is
different for different settings:
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
default one) cannot be read from the per-spider settings. These are applied
when the :class:`~scrapy.crawler.CrawlerRunner` or
:class:`~scrapy.crawler.CrawlerProcess` object is created.
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
available value is used, and if a spider requests a different reactor an
exception will be raised. These are applied when the reactor is installed.
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
the first available value is used. These are applied when the reactor is
started.
.. seealso:: :ref:`run-from-script`.
.. _distributed-crawls:
@ -243,7 +269,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
`Crawlera`_
`Zyte Smart Proxy Manager`_
If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
@ -253,5 +279,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

321
docs/topics/request-response.rst
View File

@ -26,39 +26,43 @@ Request objects
.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
a :class:`Response`.
:param url: the URL of this request
:type url: string
If the URL is invalid, a :exc:`ValueError` exception is raised.
:type url: str
:param callback: the function that will be called with the response of this
request (once its downloaded) as its first parameter. For more information
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.spiders.Spider.parse` method will be used.
:meth:`~scrapy.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
:type callback: callable
:type callback: collections.abc.Callable
:param method: the HTTP method of this request. Defaults to ``'GET'``.
:type method: string
:type method: str
:param meta: the initial values for the :attr:`Request.meta` attribute. If
given, the dict passed in this parameter will be shallow copied.
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
:type body: str or unicode
:param body: the request body. If a string is passed, then it's encoded as
bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty bytes object is stored. Regardless of the
type of this argument, the final value stored will be a bytes object
(never a string or ``None``).
:type body: bytes or str
:param headers: the headers of this request. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers). If
``None`` is passed as value, the HTTP header will not be sent at all.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
:type headers: dict
:param cookies: the request cookies. These can be sent in two forms.
@ -88,7 +92,7 @@ Request objects
To create a request that does not send stored cookies and does not
store received cookies, set the ``dont_merge_cookies`` key to ``True``
in :attr:`request.meta <scrapy.http.Request.meta>`.
in :attr:`request.meta <scrapy.Request.meta>`.
Example of a request that sends manually-defined cookies and ignores
cookie storage::
@ -100,12 +104,18 @@ Request objects
)
For more info see :ref:`cookies-mw`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
:type cookies: dict or list
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to ``str`` (if given as ``unicode``).
:type encoding: string
body to bytes (if given as a string).
:type encoding: str
:param priority: the priority of this request (defaults to ``0``).
The priority is used by the scheduler to define the order used to process
@ -117,7 +127,7 @@ Request objects
the scheduler. This is used when you want to perform an identical
request multiple times, to ignore the duplicates filter. Use it with
care, or you will get into crawling loops. Default to ``False``.
:type dont_filter: boolean
:type dont_filter: bool
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
@ -125,7 +135,11 @@ Request objects
:exc:`~twisted.python.failure.Failure` as first parameter.
For more information,
see :ref:`topics-request-response-ref-errbacks` below.
:type errback: callable
.. versionchanged:: 2.0
The *callback* parameter is no longer required when the *errback*
parameter is specified.
:type errback: collections.abc.Callable
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
@ -153,7 +167,7 @@ Request objects
.. attribute:: Request.body
A str that contains the request body.
The request body as bytes.
This attribute is read-only. To change the body of a Request use
:meth:`replace`.
@ -168,9 +182,9 @@ Request objects
See :ref:`topics-request-meta` for a list of special meta keys
recognized by Scrapy.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.meta`` attribute.
.. attribute:: Request.cb_kwargs
@ -179,11 +193,15 @@ Request objects
for new Requests, which means by default callbacks only get a :class:`Response`
object as argument.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.cb_kwargs`` attribute.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
.. _shallow copied: https://docs.python.org/2/library/copy.html
In case of a failure to process the request, this dict can be accessed as
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. autoattribute:: Request.attributes
.. method:: Request.copy()
@ -200,6 +218,15 @@ Request objects
.. automethod:: from_curl
.. automethod:: to_dict
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.utils.request.request_from_dict
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@ -273,7 +300,7 @@ errors if needed::
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"http://www.httphttpbinbin.org/", # DNS error expected
"https://example.invalid/", # DNS error expected
]
def start_requests(self):
@ -308,6 +335,31 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
----------------------------------------------
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``::
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url))
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs['main_url'],
)
.. _topics-request-meta:
Request.meta special keys
@ -318,26 +370,26 @@ are some special keys recognized by Scrapy and its built-in extensions.
Those are:
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`bindaddress`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`download_latency`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_timeout`
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`max_retry_times`
* :reqmeta:`proxy`
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`download_timeout`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_latency`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`proxy`
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* :reqmeta:`referrer_policy`
* :reqmeta:`max_retry_times`
.. reqmeta:: bindaddress
@ -381,6 +433,51 @@ The meta key is used set retry times per request. When initialized, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. _topics-stop-response-download:
Stopping the download of a Response
===================================
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signals will stop the download of a given response. See the following example::
import scrapy
class StopSpider(scrapy.Spider):
name = "stop"
start_urls = ["https://docs.scrapy.org/en/latest/"]
@classmethod
def from_crawler(cls, crawler):
spider = super().from_crawler(crawler)
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
return spider
def parse(self, response):
# 'last_chars' show that the full response was not downloaded
yield {"len": len(response.text), "last_chars": response.text[-40:]}
def on_bytes_received(self, data, request, spider):
raise scrapy.exceptions.StopDownload(fail=False)
which produces the following output::
2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened
2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for <GET https://docs.scrapy.org/en/latest/> from signal handler StopSpider.on_bytes_received
2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://docs.scrapy.org/en/latest/> (referer: None) ['download_stopped']
2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/>
{'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n \n <title>Scr'}
2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished)
By default, resulting responses are handled by their corresponding errbacks. To
call their callback instead, like in this example, pass ``fail=False`` to the
:exc:`~scrapy.exceptions.StopDownload` exception.
.. _topics-request-response-ref-request-subclasses:
Request subclasses
@ -398,7 +495,9 @@ fields with form data from :class:`Response` objects.
.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
.. class:: FormRequest(url, [formdata, ...])
.. class:: scrapy.http.request.form.FormRequest
.. class:: scrapy.http.FormRequest
.. class:: scrapy.FormRequest(url, [formdata, ...])
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
@ -407,7 +506,7 @@ fields with form data from :class:`Response` objects.
:param formdata: is a dictionary (or iterable of (key, value) tuples)
containing HTML Form data which will be url-encoded and assigned to the
body of the request.
:type formdata: dict or iterable of tuples
:type formdata: dict or collections.abc.Iterable
The :class:`FormRequest` objects support the following class method in
addition to the standard :class:`Request` methods:
@ -439,20 +538,20 @@ fields with form data from :class:`Response` objects.
:type response: :class:`Response` object
:param formname: if given, the form with name attribute set to this value will be used.
:type formname: string
:type formname: str
:param formid: if given, the form with id attribute set to this value will be used.
:type formid: string
:type formid: str
:param formxpath: if given, the first form that matches the xpath will be used.
:type formxpath: string
:type formxpath: str
:param formcss: if given, the first form that matches the css selector will be used.
:type formcss: string
:type formcss: str
:param formnumber: the number of form to use, when the response contains
multiple forms. The first one (and also the default) is ``0``.
:type formnumber: integer
:type formnumber: int
:param formdata: fields to override in the form data. If a field was
already present in the response ``<form>`` element, its value is
@ -470,23 +569,11 @@ fields with form data from :class:`Response` objects.
:param dont_click: If True, the form data will be submitted without
clicking in any element.
:type dont_click: boolean
:type dont_click: bool
The other parameters of this class method are passed directly to the
:class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
.. versionadded:: 0.17
The ``formxpath`` parameter.
.. versionadded:: 1.1.0
The ``formcss`` parameter.
.. versionadded:: 1.1.0
The ``formid`` parameter.
Request usage examples
----------------------
@ -558,13 +645,13 @@ dealing with JSON requests.
if :attr:`Request.body` argument is provided this parameter will be ignored.
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
set to ``'POST'`` automatically.
:type data: JSON serializable object
:type data: object
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
data into JSON format.
:type dumps_kwargs: dict
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
.. autoattribute:: JsonRequest.attributes
JsonRequest usage example
-------------------------
@ -583,20 +670,17 @@ Response objects
.. autoclass:: Response
A :class:`Response` object represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
:param url: the URL of this response
:type url: string
:type url: str
:param status: the HTTP status of the response. Defaults to ``200``.
:type status: integer
:type status: int
:param headers: the headers of this response. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
:param body: the response body. To access the decoded text as str you can use
:param body: the response body. To access the decoded text as a string, use
``response.text`` from an encoding-aware
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
such as :class:`TextResponse`.
@ -609,11 +693,27 @@ Response objects
:param request: the initial value of the :attr:`Response.request` attribute.
This represents the :class:`Request` that generated this response.
:type request: scrapy.http.Request
:type request: scrapy.Request
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
:param protocol: The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1", "h2"
:type protocol: :class:`str`
.. versionadded:: 2.0.0
The ``certificate`` parameter.
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. versionadded:: 2.5.0
The ``protocol`` parameter.
.. attribute:: Response.url
A string containing the URL of the response.
@ -638,10 +738,10 @@ Response objects
.. attribute:: Response.body
The body of this Response. Keep in mind that Response.body
is always a bytes object. If you want the unicode version use
:attr:`TextResponse.text` (only available in :class:`TextResponse`
and subclasses).
The response body as bytes.
If you want the body as a string, use :attr:`TextResponse.text` (only
available in :class:`TextResponse` and subclasses).
This attribute is read-only. To change the body of a Response use
:meth:`replace`.
@ -677,6 +777,8 @@ Response objects
.. attribute:: Response.cb_kwargs
.. versionadded:: 2.0
A shortcut to the :attr:`Request.cb_kwargs` attribute of the
:attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
@ -696,11 +798,36 @@ Response objects
.. attribute:: Response.certificate
.. versionadded:: 2.0.0
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
Only populated for ``https`` responses, ``None`` otherwise.
.. attribute:: Response.ip_address
.. versionadded:: 2.1.0
The IP address of the server from which the Response originated.
This attribute is currently only populated by the HTTP 1.1 download
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. attribute:: Response.protocol
.. versionadded:: 2.5.0
The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1"
This attribute is currently only populated by the HTTP download
handlers, i.e. for ``http(s)`` responses. For other handlers,
:attr:`protocol` is always ``None``.
.. autoattribute:: Response.attributes
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@ -716,18 +843,16 @@ Response objects
Constructs an absolute url by combining the Response's :attr:`url` with
a possible relative url.
This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
making this call::
urlparse.urljoin(response.url, url)
urllib.parse.urljoin(response.url, url)
.. automethod:: Response.follow
.. automethod:: Response.follow_all
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
.. _topics-request-response-ref-response-subclasses:
Response subclasses
@ -750,18 +875,18 @@ TextResponse objects
is the same as for the :class:`Response` class and is not documented here.
:param encoding: is a string which contains the encoding to use for this
response. If you create a :class:`TextResponse` object with a unicode
body, it will be encoded using this encoding (remember the body attribute
is always a string). If ``encoding`` is ``None`` (default value), the
encoding will be looked up in the response headers and body instead.
:type encoding: string
response. If you create a :class:`TextResponse` object with a string as
body, it will be converted to bytes encoded using this encoding. If
*encoding* is ``None`` (default), the encoding will be looked up in the
response headers and body instead.
:type encoding: str
:class:`TextResponse` objects support the following attributes in addition
to the standard :class:`Response` ones:
.. attribute:: TextResponse.text
Response body, as unicode.
Response body, as a string.
The same as ``response.body.decode(response.encoding)``, but the
result is cached after the first call, so you can access
@ -769,9 +894,11 @@ TextResponse objects
.. note::
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
(typically ``ascii``) instead of the response encoding.
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. attribute:: TextResponse.encoding
@ -794,9 +921,11 @@ TextResponse objects
.. attribute:: TextResponse.selector
A :class:`~scrapy.selector.Selector` instance using the response as
A :class:`~scrapy.Selector` instance using the response as
target. The selector is lazily instantiated on first access.
.. autoattribute:: TextResponse.attributes
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
@ -816,10 +945,10 @@ TextResponse objects
.. automethod:: TextResponse.follow_all
.. method:: TextResponse.body_as_unicode()
.. automethod:: TextResponse.json()
The same as :attr:`text`, but available as a method. This method is
kept for backward compatibility; please prefer ``response.text``.
Returns a Python object from deserialized JSON document.
The result is cached after the first call.
HtmlResponse objects

34
docs/topics/scheduler.rst Normal file
View File

@ -0,0 +1,34 @@
.. _topics-scheduler:
=========
Scheduler
=========
.. module:: scrapy.core.scheduler
The scheduler component receives requests from the :ref:`engine <component-engine>`
and stores them into persistent and/or non-persistent data structures.
It also gets those requests and feeds them back to the engine when it
asks for a next request to be downloaded.
Overriding the default scheduler
================================
You can use your own custom scheduler class by supplying its full
Python path in the :setting:`SCHEDULER` setting.
Minimal scheduler interface
===========================
.. autoclass:: BaseScheduler
:members:
Default Scrapy scheduler
========================
.. autoclass:: Scheduler
:members:
:special-members: __len__

45
docs/topics/selectors.rst
View File

@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is
to extract data from the HTML source. There are several libraries available to
achieve this, such as:
* `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
- `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on `ElementTree`_. (lxml is not part of the Python standard
library.)
- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python
standard library.)
Scrapy comes with its own mechanism for extracting data. They're called
selectors because they "select" certain parts of the HTML document specified
@ -36,7 +36,6 @@ defines selectors to associate those styles with specific HTML elements.
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: https://lxml.de/
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
.. _XPath: https://www.w3.org/TR/xpath/all/
.. _CSS: https://www.w3.org/TR/selectors
.. _parsel: https://parsel.readthedocs.io/en/latest/
@ -49,7 +48,7 @@ Constructing selectors
.. highlight:: python
Response objects expose a :class:`~scrapy.selector.Selector` instance
Response objects expose a :class:`~scrapy.Selector` instance
on ``.selector`` attribute:
>>> response.selector.xpath('//span/text()').get()
@ -63,9 +62,10 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
>>> response.css('span::text').get()
'good'
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
Scrapy selectors are instances of :class:`~scrapy.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
markup as an unicode string (in ``text`` argument).
markup as a string (in ``text`` argument).
Usually there is no need to construct Scrapy selectors manually:
``response`` object is available in Spider callbacks, so in most cases
it is more convenient to use ``response.css()`` and ``response.xpath()``
@ -175,7 +175,7 @@ of ``None``:
'not-found'
Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
using ``.attrib`` property of a :class:`~scrapy.Selector`:
>>> [img.attrib['src'] for img in response.css('img')]
['image1_thumb.jpg',
@ -328,8 +328,9 @@ too. Here's an example:
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
... print('Link number %d points to url %r and image %r' % args)
... href_xpath = link.xpath('@href').get()
... img_xpath = link.xpath('img/@src').get()
... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}')
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
@ -382,9 +383,9 @@ ID, or when selecting an unique element on a page):
Using selectors with regular expressions
----------------------------------------
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
:class:`~scrapy.Selector` also has a ``.re()`` method for extracting
data using regular expressions. However, unlike using ``.xpath()`` or
``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
``.css()`` methods, ``.re()`` returns a list of strings. So you
can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
@ -463,10 +464,10 @@ effectively. If you are not much familiar with XPath yet,
you may want to take a look first at this `XPath tutorial`_.
.. note::
Some of the tips are based on `this post from ScrapingHub's blog`_.
Some of the tips are based on `this post from Zyte's blog`_.
.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/
.. _topics-selectors-relative-xpaths:
@ -735,7 +736,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
Example selecting links in list item with a "class" attribute ending with a digit:
>>> from scrapy import Selector
>>> doc = u"""
>>> doc = """
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
@ -766,7 +767,7 @@ extracting text elements for example.
Example extracting microdata (sample content taken from https://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
>>> doc = u"""
>>> doc = """
... <div itemscope itemtype="http://schema.org/Product">
... <span itemprop="name">Kenmore White 17" Microwave</span>
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@ -822,7 +823,7 @@ with groups of itemscopes and corresponding itemprops::
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
... print(" properties: %s" % (props.getall()))
... print(f" properties: {props.getall()}")
... print("")
current scope: ['http://schema.org/Product']
@ -990,7 +991,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
sel.xpath("//h1")
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of unicode strings::
returning a list of strings::
sel.xpath("//h1").getall() # this includes the h1 tag
sel.xpath("//h1/text()").getall() # this excludes the h1 tag

377
docs/topics/settings.rst
View File

@ -26,9 +26,7 @@ do this by using an environment variable, ``SCRAPY_SETTINGS_MODULE``.
The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
``myproject.settings``. Note that the settings module should be on the
Python `import search path`_.
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
Python :ref:`import search path <tut-searchpath>`.
.. _populating-settings:
@ -69,7 +67,7 @@ Example::
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
own settings that will take precedence and override the project ones. They can
do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute::
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
class MySpider(scrapy.Spider):
name = 'myspider'
@ -100,6 +98,32 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Import paths and classes
========================
.. versionadded:: 2.4.0
When a setting references a callable object to be imported by Scrapy, such as a
class or a function, there are two different ways you can specify that object:
- As a string containing the import path of that object
- As the object itself
For example::
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
}
.. note:: Passing non-callable objects is not supported.
How to access settings
======================
@ -112,19 +136,19 @@ In a spider, the settings are available through ``self.settings``::
start_urls = ['http://example.com']
def parse(self, response):
print("Existing settings: %s" % self.settings.attributes.keys())
print(f"Existing settings: {self.settings.attributes.keys()}")
.. note::
The ``settings`` attribute is set in the base Spider class after the spider
is initialized. If you want to use the settings before the initialization
(e.g., in your spider's ``__init__()`` method), you'll need to override the
:meth:`~scrapy.spiders.Spider.from_crawler` method.
:meth:`~scrapy.Spider.from_crawler` method.
Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
extensions, middlewares and item pipelines::
class MyExtension(object):
class MyExtension:
def __init__(self, log_is_enabled=False):
if log_is_enabled:
print("log is enabled!")
@ -180,6 +204,19 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
.. setting:: AWS_SESSION_TOKEN
AWS_SESSION_TOKEN
-----------------
Default: ``None``
The AWS security token used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
`temporary security credentials`_.
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
.. setting:: AWS_ENDPOINT_URL
AWS_ENDPOINT_URL
@ -218,6 +255,32 @@ Default: ``None``
The name of the region associated with the AWS client.
.. setting:: ASYNCIO_EVENT_LOOP
ASYNCIO_EVENT_LOOP
------------------
Default: ``None``
Import path of a given ``asyncio`` event loop class.
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
asyncio event loop to be used with it. Set the setting to the import path of the
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
event loop will be used.
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
class to be used.
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
.. caution:: Please be aware that, when using a non-default event loop
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
:func:`asyncio.set_event_loop`, which will set the specified event loop
as the current loop for the current OS thread.
.. setting:: BOT_NAME
BOT_NAME
@ -238,8 +301,8 @@ CONCURRENT_ITEMS
Default: ``100``
Maximum number of concurrent items (per response) to process in parallel in the
Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
Maximum number of concurrent items (per response) to process in parallel in
:ref:`item pipelines <topics-item-pipeline>`.
.. setting:: CONCURRENT_REQUESTS
@ -288,7 +351,7 @@ is non-zero, download delay is enforced per IP, not per domain.
DEFAULT_ITEM_CLASS
------------------
Default: ``'scrapy.item.Item'``
Default: ``'scrapy.Item'``
The default class that will be used for instantiating items in the :ref:`the
Scrapy shell <topics-shell>`.
@ -308,6 +371,11 @@ Default::
The default headers used for Scrapy HTTP Requests. They're populated in the
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
.. setting:: DEPTH_LIMIT
DEPTH_LIMIT
@ -329,8 +397,8 @@ Default: ``0``
Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
a :class:`~scrapy.http.Request` based on its depth.
An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
a :class:`~scrapy.Request` based on its depth.
The priority of a request is adjusted as follows::
@ -381,6 +449,8 @@ DNS in-memory cache size.
DNS_RESOLVER
------------
.. versionadded:: 2.0
Default: ``'scrapy.resolver.CachingThreadedResolver'``
The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
@ -420,10 +490,9 @@ connections (for ``HTTP10DownloadHandler``).
.. note::
HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
unless you use Twisted<11.1, or if you really want to use HTTP/1.0
and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
accordingly, i.e. to
``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
unless you really want to use HTTP/1.0 and override
:setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
@ -447,7 +516,6 @@ or even enable client-side authentication (and various other things).
Scrapy also has another context factory class that you can set,
``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
which uses the platform's certificates to validate remote endpoints.
**This is only available if you use Twisted>=14.0.**
If you do use a custom ContextFactory, make sure its ``__init__`` method
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@ -471,7 +539,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
specific cipher that is not included in ``DEFAULT`` if a website requires it.
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@ -494,10 +562,6 @@ This setting must be one of these string values:
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
.. note::
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
or above (Twisted>=14.0 if you can).
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -606,6 +670,7 @@ DOWNLOAD_HANDLERS_BASE
Default::
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@ -626,6 +691,45 @@ handler (without replacement), place this in your ``settings.py``::
'ftp': None,
}
.. _http2:
The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
}
.. warning::
HTTP/2 support in Scrapy is experimental, and not yet recommended for
production environments. Future Scrapy versions may introduce related
changes without a deprecation period or warning.
.. note::
Known limitations of the current HTTP/2 implementation of Scrapy include:
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).
- No setting to specify a maximum `frame size`_ larger than the default
value, 16384. Connections to servers that send a larger frame will
fail.
- No support for `server pushes`_, which are ignored.
- No support for the :signal:`bytes_received` and
:signal:`headers_received` signals.
.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@ -660,8 +764,6 @@ If you want to disable it set to 0.
spider attribute and per-request using :reqmeta:`download_maxsize`
Request.meta key.
This feature needs Twisted >= 11.1.
.. setting:: DOWNLOAD_WARNSIZE
DOWNLOAD_WARNSIZE
@ -679,8 +781,6 @@ If you want to disable it set to 0.
spider attribute and per-request using :reqmeta:`download_warnsize`
Request.meta key.
This feature needs Twisted >= 11.1.
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
DOWNLOAD_FAIL_ON_DATALOSS
@ -707,6 +807,15 @@ Optionally, this can be set per-request basis by using the
If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
.. warning::
This setting is ignored by the
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
error, the corresponding HTTP/2 connection may be corrupted, affecting other
requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
failure is always raised for every request that was using that connection.
.. setting:: DUPEFILTER_CLASS
DUPEFILTER_CLASS
@ -720,14 +829,14 @@ The default (``RFPDupeFilter``) filters based on request fingerprint using
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
override its ``request_fingerprint`` method. This method should accept
scrapy :class:`~scrapy.http.Request` object and return its fingerprint
scrapy :class:`~scrapy.Request` object and return its fingerprint
(a string).
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
Be very careful about this however, because you can get into crawling loops.
It's usually a better idea to set the ``dont_filter`` parameter to
``True`` on the specific :class:`~scrapy.http.Request` that should not be
``True`` on the specific :class:`~scrapy.Request` that should not be
filtered.
.. setting:: DUPEFILTER_DEBUG
@ -796,6 +905,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
:ref:`Amazon S3 <topics-feed-storage-s3>`.
.. setting:: FEED_STORAGE_GCS_ACL
FEED_STORAGE_GCS_ACL
--------------------
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
.. setting:: FTP_PASSIVE_MODE
FTP_PASSIVE_MODE
@ -835,6 +952,15 @@ Default: ``"anonymous"``
The username to use for FTP connections when there is no ``"ftp_user"``
in ``Request`` meta.
.. setting:: GCS_PROJECT_ID
GCS_PROJECT_ID
-----------------
Default: ``None``
The Project ID that will be used when storing data on `Google Cloud Storage`_.
.. setting:: ITEM_PIPELINES
ITEM_PIPELINES
@ -863,6 +989,16 @@ Default: ``{}``
A dict containing the pipelines enabled by default in Scrapy. You should never
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
.. setting:: JOBDIR
JOBDIR
------
Default: ``''``
A string indicating the directory for storing the state of a crawl when
:ref:`pausing and resuming crawls <topics-jobs>`.
.. setting:: LOG_ENABLED
LOG_ENABLED
@ -890,6 +1026,16 @@ Default: ``None``
File name to use for logging output. If ``None``, standard error will be used.
.. setting:: LOG_FILE_APPEND
LOG_FILE_APPEND
---------------
Default: ``True``
If ``False``, the log file specified with :setting:`LOG_FILE` will be
overwritten (discarding the output from previous runs, if any).
.. setting:: LOG_FORMAT
LOG_FORMAT
@ -897,10 +1043,9 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
placeholders.
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
list of available placeholders.
.. setting:: LOG_DATEFORMAT
@ -910,10 +1055,9 @@ LOG_DATEFORMAT
Default: ``'%Y-%m-%d %H:%M:%S'``
String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
directives.
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
in :setting:`LOG_FORMAT`. Refer to the
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
whole list of available directives.
.. setting:: LOG_FORMATTER
@ -1025,8 +1169,6 @@ See :ref:`topics-extensions-ref-memusage`.
MEMUSAGE_CHECK_INTERVAL_SECONDS
-------------------------------
.. versionadded:: 1.1
Default: ``60.0``
Scope: ``scrapy.extensions.memusage``
@ -1114,17 +1256,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
this value if you're experiencing problems with insufficient blocking IO.
.. setting:: REDIRECT_MAX_TIMES
REDIRECT_MAX_TIMES
------------------
Default: ``20``
Defines the maximum times a request can be redirected. After this maximum the
request's response is returned as is. We used Firefox default value for the
same task.
.. setting:: REDIRECT_PRIORITY_ADJUST
REDIRECT_PRIORITY_ADJUST
@ -1139,20 +1270,6 @@ Adjust redirect request priority relative to original request:
- **a positive priority adjust (default) means higher priority.**
- a negative priority adjust means lower priority.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. setting:: ROBOTSTXT_OBEY
ROBOTSTXT_OBEY
@ -1200,7 +1317,8 @@ SCHEDULER
Default: ``'scrapy.core.scheduler.Scheduler'``
The scheduler to use for crawling.
The scheduler class to be used for crawling.
See the :ref:`topics-scheduler` topic for details.
.. setting:: SCHEDULER_DEBUG
@ -1258,6 +1376,9 @@ does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
SCRAPER_SLOT_MAX_ACTIVE_SIZE
----------------------------
.. versionadded:: 2.0
Default: ``5_000_000``
Soft limit (in bytes) for response data being processed.
@ -1315,8 +1436,6 @@ The class that will be used for loading spiders, which must implement the
SPIDER_LOADER_WARN_ONLY
-----------------------
.. versionadded:: 1.3.3
Default: ``False``
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
@ -1417,17 +1536,6 @@ Default: ``True``
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
will be enabled (provided its extension is also enabled).
.. setting:: TELNETCONSOLE_PORT
TELNETCONSOLE_PORT
------------------
Default: ``[6023, 6073]``
The port range to use for the telnet console. If set to ``None`` or ``0``, a
dynamically assigned port is used. For more info see
:ref:`topics-telnetconsole`.
.. setting:: TEMPLATES_DIR
TEMPLATES_DIR
@ -1447,24 +1555,94 @@ in the ``project`` subdirectory.
TWISTED_REACTOR
---------------
.. versionadded:: 2.0
Default: ``None``
Import path of a given Twisted reactor, for instance:
:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`.
Import path of a given :mod:`~twisted.internet.reactor`.
Scrapy will install this reactor if no other is installed yet, such as when
the ``scrapy`` CLI program is invoked or when using the
:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the
:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct
reactor manually. An exception will be raised if the installation fails.
Scrapy will install this reactor if no other reactor is installed yet, such as
when the ``scrapy`` CLI program is invoked or when using the
:class:`~scrapy.crawler.CrawlerProcess` class.
The default value for this option is currently ``None``, which means that Scrapy
will not attempt to install any specific reactor, and the default one defined by
Twisted for the current platform will be used. This is to maintain backward
compatibility and avoid possible problems caused by using a non-default reactor.
If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also
need to install the correct reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`:
For additional information, please see
:doc:`core/howto/choosing-reactor`.
.. autofunction:: scrapy.utils.reactor.install_reactor
If a reactor is already installed,
:func:`~scrapy.utils.reactor.install_reactor` has no effect.
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
:exc:`Exception` if the installed reactor does not match the
:setting:`TWISTED_REACTOR` setting; therefore, having top-level
:mod:`~twisted.internet.reactor` imports in project files and imported
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
In order to use the reactor installed by Scrapy::
import scrapy
from twisted.internet import reactor
class QuotesSpider(scrapy.Spider):
name = 'quotes'
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
which raises :exc:`Exception`, becomes::
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will install the default reactor defined by Twisted for the
current platform. This is to maintain backward compatibility and avoid possible
problems caused by using a non-default reactor.
For additional information, see :doc:`core/howto/choosing-reactor`.
.. setting:: URLLENGTH_LIMIT
@ -1476,8 +1654,19 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
The maximum URL length to allow for crawled URLs.
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
:setting:`DEPTH_LIMIT`.
Use ``0`` to allow URLs of any length.
The default value is copied from the `Microsoft Internet Explorer maximum URL
length`_, even though this setting exists for different reasons.
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
.. setting:: USER_AGENT
@ -1489,7 +1678,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
The default User-Agent to use when crawling, unless overridden. This user agent is
also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
there is no overridding User-Agent header specified for the request.
there is no overriding User-Agent header specified for the request.
Settings documented elsewhere:
@ -1500,7 +1689,7 @@ case to see how to enable and use them.
.. settingslist::
.. _Amazon web services: https://aws.amazon.com/
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
.. _Google Cloud Storage: https://cloud.google.com/storage/

58
docs/topics/shell.rst
View File

@ -95,20 +95,21 @@ convenience.
Available Shortcuts
-------------------
* ``shelp()`` - print a help with the list of available objects and shortcuts
- ``shelp()`` - print a help with the list of available objects and
shortcuts
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
URL and update all related objects accordingly. You can optionaly ask for
HTTP 3xx redirections to not be followed by passing ``redirect=False``
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
and update all related objects accordingly. You can optionally ask for HTTP
3xx redirections to not be followed by passing ``redirect=False``
* ``fetch(request)`` - fetch a new response from the given request and
update all related objects accordingly.
- ``fetch(request)`` - fetch a new response from the given request and update
all related objects accordingly.
* ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
- ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
.. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
@ -117,26 +118,26 @@ Available Scrapy objects
The Scrapy shell automatically creates some convenient objects from the
downloaded page, like the :class:`~scrapy.http.Response` object and the
:class:`~scrapy.selector.Selector` objects (for both HTML and XML
:class:`~scrapy.Selector` objects (for both HTML and XML
content).
Those objects are:
* ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
* ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.spiders.Spider` object if there is no spider found for
the current URL
- ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.Spider` object if there is no spider found for the
current URL
* ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
page. You can modify this request using :meth:`~scrapy.http.Request.replace`
or fetch a new request (without leaving the shell) using the ``fetch``
shortcut.
- ``request`` - a :class:`~scrapy.Request` object of the last fetched
page. You can modify this request using
:meth:`~scrapy.Request.replace` or fetch a new request (without
leaving the shell) using the ``fetch`` shortcut.
* ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
- ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
* ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
- ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
Example of shell session
========================
@ -156,6 +157,17 @@ First, we launch the shell::
scrapy shell 'https://scrapy.org' --nolog
.. note::
Remember to always enclose URLs in quotes when running the Scrapy shell from
the command line, otherwise URLs containing arguments (i.e. the ``&`` character)
will not work.
On Windows, use double quotes instead::
scrapy shell "https://scrapy.org" --nolog
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
list of available objects and useful shortcuts (you'll notice that these lines
all start with the ``[s]`` prefix)::

255
docs/topics/signals.rst
View File

@ -16,8 +16,7 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action:
::
Here is a simple example showing how you can catch signals and perform some action::
from scrapy import signals
from scrapy import Spider
@ -46,14 +45,51 @@ Here is a simple example showing how you can catch signals and perform some acti
def parse(self, response):
pass
.. _signal-deferred:
Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, see the :ref:`topics-signals-ref` below to know
which ones.
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
Let's take an example::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['http://quotes.toscrape.com/page/1/']
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
# The next item will be scraped only after
# deferred (d) is fired
return d
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
.. _topics-signals-ref:
@ -65,22 +101,25 @@ Built-in signals reference
Here's the list of Scrapy built-in signals and their meaning.
engine_started
Engine signals
--------------
engine_started
~~~~~~~~~~~~~~
.. signal:: engine_started
.. function:: engine_started()
Sent when the Scrapy engine has started crawling.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
.. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
depending on how the spider was started. So **don't** rely on this signal
getting fired before :signal:`spider_opened`.
engine_stopped
--------------
~~~~~~~~~~~~~~
.. signal:: engine_stopped
.. function:: engine_stopped()
@ -88,10 +127,21 @@ engine_stopped
Sent when the Scrapy engine is stopped (for example, when a crawling
process has finished).
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
Item signals
------------
.. note::
As at max :setting:`CONCURRENT_ITEMS` items are processed in
parallel, many deferreds are fired together using
:class:`~twisted.internet.defer.DeferredList`. Hence the next
batch waits for the :class:`~twisted.internet.defer.DeferredList`
to fire and then runs the respective item signal handler for
the next batch of scraped items.
item_scraped
------------
~~~~~~~~~~~~
.. signal:: item_scraped
.. function:: item_scraped(item, response, spider)
@ -99,19 +149,19 @@ item_scraped
Sent when an item has been scraped, after it has passed all the
:ref:`topics-item-pipeline` stages (without being dropped).
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item scraped
:type item: dict or :class:`~scrapy.item.Item` object
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param response: the response from where the item was scraped
:type response: :class:`~scrapy.http.Response` object
item_dropped
------------
~~~~~~~~~~~~
.. signal:: item_dropped
.. function:: item_dropped(item, response, exception, spider)
@ -119,13 +169,13 @@ item_dropped
Sent after an item has been dropped from the :ref:`topics-item-pipeline`
when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param response: the response from where the item was dropped
:type response: :class:`~scrapy.http.Response` object
@ -136,7 +186,7 @@ item_dropped
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
item_error
------------
~~~~~~~~~~
.. signal:: item_error
.. function:: item_error(item, response, spider, failure)
@ -144,22 +194,25 @@ item_error
Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:param item: the item that caused the error in the :ref:`topics-item-pipeline`
:type item: :ref:`item object <item-types>`
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
Spider signals
--------------
spider_closed
-------------
~~~~~~~~~~~~~
.. signal:: spider_closed
.. function:: spider_closed(spider, reason)
@ -167,10 +220,10 @@ spider_closed
Sent after a spider has been closed. This can be used to release per-spider
resources reserved on :signal:`spider_opened`.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
:param reason: a string which describes the reason why the spider was closed. If
it was closed because the spider has completed scraping, the reason
@ -182,7 +235,7 @@ spider_closed
:type reason: str
spider_opened
-------------
~~~~~~~~~~~~~
.. signal:: spider_opened
.. function:: spider_opened(spider)
@ -191,13 +244,13 @@ spider_opened
reserve per-spider resources, but can be used for any task that needs to be
performed when a spider is opened.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
spider_idle
-----------
~~~~~~~~~~~
.. signal:: spider_idle
.. function:: spider_idle(spider)
@ -215,10 +268,17 @@ spider_idle
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
This signal does not support returning deferreds from their handlers.
Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
exception to provide a custom spider closing reason. An
idle handler is the perfect place to put some code that assesses
the final spider results and update the final closing reason
accordingly (e.g. setting it to 'too_few_results' instead of
'finished').
This signal does not support returning deferreds from its handlers.
:param spider: the spider which has gone idle
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. note:: Scheduling some requests in your :signal:`spider_idle` handler does
**not** guarantee that it can prevent the spider from being closed,
@ -227,14 +287,14 @@ spider_idle
due to duplication).
spider_error
------------
~~~~~~~~~~~~
.. signal:: spider_error
.. function:: spider_error(failure, response, spider)
Sent when a spider callback generates an error (i.e. raises an exception).
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
@ -243,77 +303,145 @@ spider_error
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
Request signals
---------------
request_scheduled
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: request_scheduled
.. function:: request_scheduled(request, spider)
Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
Sent when the engine schedules a :class:`~scrapy.Request`, to be
downloaded later.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
request_dropped
---------------
~~~~~~~~~~~~~~~
.. signal:: request_dropped
.. function:: request_dropped(request, spider)
Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
Sent when a :class:`~scrapy.Request`, scheduled by the engine to be
downloaded later, is rejected by the scheduler.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
request_reached_downloader
---------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_reached_downloader
.. function:: request_reached_downloader(request, spider)
Sent when a :class:`~scrapy.http.Request` reached downloader.
Sent when a :class:`~scrapy.Request` reached downloader.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached downloader
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
request_left_downloader
-----------------------
~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_left_downloader
.. function:: request_left_downloader(request, spider)
Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
.. versionadded:: 2.0
Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of
failure.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the downloader
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
bytes_received
~~~~~~~~~~~~~~
.. versionadded:: 2.2
.. signal:: bytes_received
.. function:: bytes_received(data, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
received for a specific request. This signal might be fired multiple
times for the same request, with partial data each time. For instance,
a possible scenario for a 25 kb response would be two signals fired
with 10 kb of data, and a final one with 5 kb of data.
Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type data: :class:`bytes` object
:param request: the request that generated the download
:type request: :class:`~scrapy.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
headers_received
~~~~~~~~~~~~~~~~
.. versionadded:: 2.5
.. signal:: headers_received
.. function:: headers_received(headers, body_length, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
available for a given request, before downloading any additional content.
Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
This signal does not support returning deferreds from its handlers.
:param headers: the headers received by the download handler
:type headers: :class:`scrapy.http.headers.Headers` object
:param body_length: expected size of the response body, in bytes
:type body_length: `int`
:param request: the request that generated the download
:type request: :class:`~scrapy.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
Response signals
----------------
response_received
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: response_received
.. function:: response_received(response, request, spider)
@ -321,32 +449,37 @@ response_received
Sent when the engine receives a new :class:`~scrapy.http.Response` from the
downloader.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param response: the response received
:type response: :class:`~scrapy.http.Response` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. note:: The ``request`` argument might not contain the original request that
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
the :class:`~scrapy.http.Response` object and sets a specific ``request``
attribute.
response_downloaded
-------------------
~~~~~~~~~~~~~~~~~~~
.. signal:: response_downloaded
.. function:: response_downloaded(response, request, spider)
Sent by the downloader right after a ``HTTPResponse`` is downloaded.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param response: the response downloaded
:type response: :class:`~scrapy.http.Response` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:type request: :class:`~scrapy.Request` object
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object

53
docs/topics/spider-middleware.rst
View File

@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output(response, result, spider)
@ -102,20 +102,19 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
objects.
:class:`~scrapy.Request` objects and :ref:`item object
<topics-items>`.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.http.Request`, dict
or :class:`~scrapy.item.Item` objects
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item object <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_exception(response, exception, spider)
@ -123,8 +122,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.http.Request`, dict or
:class:`~scrapy.item.Item` objects.
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
objects.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_spider_exception` in the following
@ -140,22 +139,20 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type response: :class:`~scrapy.http.Response` object
:param exception: the exception raised
:type exception: `Exception`_ object
:type exception: :exc:`Exception` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: process_start_requests(start_requests, spider)
.. versionadded:: 0.15
This method is called with the start requests of the spider, and works
similarly to the :meth:`process_spider_output` method, except that it
doesn't have a response associated and must return only requests (not
items).
It receives an iterable (in the ``start_requests`` parameter) and must
return another iterable of :class:`~scrapy.http.Request` objects.
return another iterable of :class:`~scrapy.Request` objects.
.. note:: When implementing this method in your spider middleware, you
should always return an iterable (that follows the input one) and
@ -167,26 +164,22 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
(like a time limit or item/page count).
:param start_requests: the start requests
:type start_requests: an iterable of :class:`~scrapy.http.Request`
:type start_requests: an iterable of :class:`~scrapy.Request`
:param spider: the spider to whom the start requests belong
:type spider: :class:`~scrapy.spiders.Spider` object
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
If present, this classmethod is called to create a middleware instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the middleware. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for middleware to
access them and hook its functionality into Scrapy.
:param crawler: crawler that uses this middleware
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
.. _topics-spider-middleware-ref:
Built-in spider middleware reference
@ -258,9 +251,10 @@ this::
.. reqmeta:: handle_httpstatus_all
The ``handle_httpstatus_list`` key of :attr:`Request.meta
<scrapy.http.Request.meta>` can also be used to specify which response codes to
<scrapy.Request.meta>` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
to ``True`` if you want to allow any response code for a request.
to ``True`` if you want to allow any response code for a request, and ``False`` to
disable the effects of the ``handle_httpstatus_all`` key.
Keep in mind, however, that it's usually a bad idea to handle non-200
responses, unless you really know what you're doing.
@ -301,7 +295,7 @@ OffsiteMiddleware
Filters out Requests for URLs outside the domains covered by the spider.
This middleware filters out every request whose host names aren't in the
spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
All subdomains of any domain in the list are also allowed.
E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
but not ``www2.example.com`` nor ``example.com``.
@ -319,10 +313,10 @@ OffsiteMiddleware
will be printed (but only for the first request filtered).
If the spider doesn't define an
:attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the
:attr:`~scrapy.Spider.allowed_domains` attribute, or the
attribute is empty, the offsite middleware will allow all requests.
If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute
If the request has the :attr:`~scrapy.Request.dont_filter` attribute
set, the offsite middleware will allow the request even if its domain is not
listed in allowed domains.
@ -346,8 +340,6 @@ RefererMiddleware settings
REFERER_ENABLED
^^^^^^^^^^^^^^^
.. versionadded:: 0.15
Default: ``True``
Whether to enable referer middleware.
@ -357,8 +349,6 @@ Whether to enable referer middleware.
REFERRER_POLICY
^^^^^^^^^^^^^^^
.. versionadded:: 1.4
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy
@ -450,4 +440,3 @@ UrlLengthMiddleware
settings (see the settings documentation for more info):
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.

106
docs/topics/spiders.rst
View File

@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this:
those requests.
The first requests to perform are obtained by calling the
:meth:`~scrapy.spiders.Spider.start_requests` method which (by default)
generates :class:`~scrapy.http.Request` for the URLs specified in the
:attr:`~scrapy.spiders.Spider.start_urls` and the
:attr:`~scrapy.spiders.Spider.parse` method as callback function for the
:meth:`~scrapy.Spider.start_requests` method which (by default)
generates :class:`~scrapy.Request` for the URLs specified in the
:attr:`~scrapy.Spider.start_urls` and the
:attr:`~scrapy.Spider.parse` method as callback function for the
Requests.
2. In the callback function, you parse the response (web page) and return either
dicts with extracted data, :class:`~scrapy.item.Item` objects,
:class:`~scrapy.http.Request` objects, or an iterable of these objects.
2. In the callback function, you parse the response (web page) and return
:ref:`item objects <topics-items>`,
:class:`~scrapy.Request` objects, or an iterable of these objects.
Those Requests will also contain a callback (maybe
the same) and will then be downloaded by Scrapy and then their
response handled by the specified callback.
@ -50,7 +50,8 @@ We will talk about those types here.
scrapy.Spider
=============
.. class:: Spider()
.. class:: scrapy.spiders.Spider()
.. class:: scrapy.Spider()
This is the simplest spider, and the one from which every other spider
must inherit (including spiders that come bundled with Scrapy, as well as spiders
@ -86,7 +87,7 @@ scrapy.Spider
A list of URLs where the spider will begin to crawl from, when no
particular URLs are specified. So, the first pages downloaded will be those
listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data
contained in the start URLs.
.. attribute:: custom_settings
@ -121,7 +122,12 @@ scrapy.Spider
send log messages through it as described on
:ref:`topics-logging-from-spiders`.
.. method:: from_crawler(crawler, \*args, \**kwargs)
.. attribute:: state
A dict you can use to persist some spider state between batches.
See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
.. method:: from_crawler(crawler, *args, **kwargs)
This is the class method used by Scrapy to create your spiders.
@ -179,8 +185,8 @@ scrapy.Spider
the same requirements as the :class:`Spider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.http.Request` and/or
dicts or :class:`~scrapy.item.Item` objects.
iterable of :class:`~scrapy.Request` and/or :ref:`item objects
<topics-items>`.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`
@ -234,7 +240,7 @@ Return multiple Requests and items from a single callback::
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :ref:`topics-items`::
to give data more structure you can use :class:`~scrapy.Item` objects::
import scrapy
from myproject.items import MyItem
@ -279,7 +285,7 @@ Spiders can access arguments in their `__init__` methods::
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = ['http://www.example.com/categories/%s' % category]
self.start_urls = [f'http://www.example.com/categories/{category}']
# ...
The default `__init__` method will take any spider arguments
@ -292,15 +298,21 @@ The above example can also be written as follows::
name = 'myspider'
def start_requests(self):
yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
If you were to set the ``start_urls`` attribute from the command line,
you would have to parse it on your own into a list
using something like
`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
using something like :func:`ast.literal_eval` or :func:`json.loads`
and then set it as an attribute.
Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)
@ -360,13 +372,14 @@ CrawlSpider
described below. If multiple rules match the same link, the first one
will be used, according to the order they're defined in this attribute.
This spider also exposes an overrideable method:
This spider also exposes an overridable method:
.. method:: parse_start_url(response)
.. method:: parse_start_url(response, **kwargs)
This method is called for the start_urls responses. It allows to parse
This method is called for each response produced for the URLs in
the spider's ``start_urls`` attribute. It allows to parse
the initial responses and must return either an
:class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
:ref:`item object <topics-items>`, a :class:`~scrapy.Request`
object, or an iterable containing any of them.
Crawling rules
@ -376,7 +389,7 @@ Crawling rules
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
defines how links will be extracted from each crawled page. Each produced link will
be used to generate a :class:`~scrapy.http.Request` object, which will contain the
be used to generate a :class:`~scrapy.Request` object, which will contain the
link's text in its ``meta`` dictionary (under the ``link_text`` key).
If omitted, a default link extractor created with no arguments will be used,
resulting in all links being extracted.
@ -385,16 +398,11 @@ Crawling rules
object with that name will be used) to be called for each link extracted with
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
as its first argument and must return either a single instance or an iterable of
:class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
:ref:`item objects <topics-items>` and/or :class:`~scrapy.Request` objects
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
object will contain the text of the link that produced the :class:`~scrapy.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
callback, since the :class:`CrawlSpider` uses the ``parse`` method
itself to implement its logic. So if you override the ``parse`` method,
the crawl spider will no longer work.
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
callback function.
@ -409,7 +417,7 @@ Crawling rules
``process_request`` is a callable (or a string, in which case a method from
the spider object with that name will be used) which will be called for every
:class:`~scrapy.http.Request` extracted by this rule. This callable should
:class:`~scrapy.Request` extracted by this rule. This callable should
take said request as first argument and the :class:`~scrapy.http.Response`
from which the request originated as second argument. It must return a
``Request`` object or ``None`` (to filter out the request).
@ -420,6 +428,13 @@ Crawling rules
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
unexpected behaviour can occur otherwise.
.. versionadded:: 2.0
The *errback* parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
@ -450,13 +465,18 @@ Let's now take a look at an example CrawlSpider with rules::
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
url = response.xpath('//td[@id="additional_data"]/@href').get()
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
def parse_additional_page(self, response, item):
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
return item
This spider would start crawling example.com's home page, collecting category
links, and item links, parsing the latter with the ``parse_item`` method. For
each item response, some data will be extracted from the HTML using XPath, and
an :class:`~scrapy.item.Item` will be filled with it.
an :class:`~scrapy.Item` will be filled with it.
XMLFeedSpider
-------------
@ -479,11 +499,11 @@ XMLFeedSpider
- ``'iternodes'`` - a fast iterator based on regular expressions
- ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
Keep in mind this uses DOM parsing and must load all DOM in memory
which could be a problem for big feeds
- ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
- ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`.
Keep in mind this uses DOM parsing and must load all DOM in memory
which could be a problem for big feeds
@ -501,7 +521,7 @@ XMLFeedSpider
available in that document that will be processed with this spider. The
``prefix`` and ``uri`` will be used to automatically register
namespaces using the
:meth:`~scrapy.selector.Selector.register_namespace` method.
:meth:`~scrapy.Selector.register_namespace` method.
You can then specify nodes with namespaces in the :attr:`itertag`
attribute.
@ -514,7 +534,7 @@ XMLFeedSpider
itertag = 'n:url'
# ...
Apart from these new attributes, this spider has the following overrideable
Apart from these new attributes, this spider has the following overridable
methods too:
.. method:: adapt_response(response)
@ -528,10 +548,10 @@ XMLFeedSpider
This method is called for the nodes matching the provided tag name
(``itertag``). Receives the response and an
:class:`~scrapy.selector.Selector` for each node. Overriding this
:class:`~scrapy.Selector` for each node. Overriding this
method is mandatory. Otherwise, you spider won't work. This method
must return either a :class:`~scrapy.item.Item` object, a
:class:`~scrapy.http.Request` object, or an iterable containing any of
must return an :ref:`item object <topics-items>`, a
:class:`~scrapy.Request` object, or an iterable containing any of
them.
.. method:: process_results(response, results)
@ -540,7 +560,11 @@ XMLFeedSpider
spider, and it's intended to perform any last time processing required
before returning the results to the framework core, for example setting the
item IDs. It receives a list of results and the response which originated
those results. It must return a list of results (Items or Requests).
those results. It must return a list of results (items or requests).
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
unexpected behaviour can occur otherwise.
XMLFeedSpider example
@ -569,7 +593,7 @@ These spiders are pretty easy to use, let's have a look at one example::
Basically what we did up there was to create a spider that downloads a feed from
the given ``start_urls``, and then iterates through each of its ``item`` tags,
prints them out, and stores some random data in an :class:`~scrapy.item.Item`.
prints them out, and stores some random data in an :class:`~scrapy.Item`.
CSVFeedSpider
-------------

2
docs/topics/stats.rst
View File

@ -32,7 +32,7 @@ Common Stats Collector uses
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
class ExtensionThatAccessStats(object):
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats

14
docs/topics/telnetconsole.rst
View File

@ -40,10 +40,10 @@ the console you need to type::
Connected to localhost.
Escape character is '^]'.
Username:
Password:
Password:
>>>
By default Username is ``scrapy`` and Password is autogenerated. The
By default Username is ``scrapy`` and Password is autogenerated. The
autogenerated Password can be seen on Scrapy logs like the example below::
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
@ -63,7 +63,7 @@ Available variables in the telnet console
=========================================
The telnet console is like a regular Python shell running inside the Scrapy
process, so you can do anything from it including importing new modules, etc.
process, so you can do anything from it including importing new modules, etc.
However, the telnet console comes with some default variables defined for
convenience:
@ -89,13 +89,11 @@ convenience:
+----------------+-------------------------------------------------------------------+
| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
| ``p`` | a shortcut to the `pprint.pprint`_ function |
| ``p`` | a shortcut to the :func:`pprint.pprint` function |
+----------------+-------------------------------------------------------------------+
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
Telnet console usage examples
=============================
@ -112,11 +110,10 @@ using the telnet console::
Execution engine status
time()-engine.start_time : 8.62972998619
engine.has_capacity() : False
len(engine.downloader.active) : 16
engine.scraper.is_idle() : False
engine.spider.name : followall
engine.spider_is_idle(engine.spider) : False
engine.spider_is_idle() : False
engine.slot.closing : False
len(engine.slot.inprogress) : 16
len(engine.slot.scheduler.dqs or []) : 0
@ -208,4 +205,3 @@ Default: ``None``
The password used for the telnet console, default behaviour is to have it
autogenerated

85
docs/utils/linkfix.py
View File

@ -14,50 +14,57 @@ Author: dufferzafar
import re
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
def main():
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# For every line, fix the respective file
for line in output_lines:
match = re.match(line_re, line)
# A regex that matches standard linkcheck output lines
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
if match:
newfilename = match.group(1)
errortype = match.group(2)
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
# Broken links can't be fixed and
# I am not sure what do with the local ones.
if errortype.lower() in ["broken", "local"]:
print("Not Fixed: " + line)
# For every line, fix the respective file
for line in output_lines:
match = re.match(line_re, line)
if match:
newfilename = match.group(1)
errortype = match.group(2)
# Broken links can't be fixed and
# I am not sure what do with the local ones.
if errortype.lower() in ["broken", "local"]:
print("Not Fixed: " + line)
else:
# If this is a new file
if newfilename != _filename:
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4))
else:
# If this is a new file
if newfilename != _filename:
# We don't understand what the current line means!
print("Not Understood: " + line)
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4))
else:
# We don't understand what the current line means!
print("Not Understood: " + line)
if __name__ == '__main__':
main()

24
docs/versioning.rst
View File

@ -1,7 +1,7 @@
.. _versioning:
============================
Versioning and API Stability
Versioning and API stability
============================
Versioning
@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
large changes.
* *B* is the release number. This will include many changes including features
and things that possibly break backward compatibility, although we strive to
keep theses cases at a minimum.
keep these cases at a minimum.
* *C* is the bugfix release number.
Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,
@ -34,7 +34,7 @@ For example:
production)
API Stability
API stability
=============
API stability was one of the major goals for the *1.0* release.
@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
same way.
.. _deprecation-policy:
Deprecation policy
==================
We aim to maintain support for deprecated Scrapy features for at least 1 year.
For example, if a feature is deprecated in a Scrapy version released on
June 15th 2020, that feature should continue to work in versions released on
June 14th 2021 or before that.
Any new Scrapy release after a year *may* remove support for that deprecated
feature.
All deprecated features removed in a Scrapy release are explicitly mentioned in
the :ref:`release notes <news>`.
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases

2
extras/qps-bench-server.py
View File

@ -37,7 +37,7 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
if 'latency' in request.args:
latency = float(request.args['latency'][0])

8
extras/qpsclient.py
View File

@ -1,5 +1,5 @@
"""
A spider that generate light requests to meassure QPS troughput
A spider that generate light requests to meassure QPS throughput
usage:
@ -27,7 +27,7 @@ class QPSSpider(Spider):
slots = 1
def __init__(self, *a, **kw):
super(QPSSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += '?latency={0}'.format(self.latency)
url += f'?latency={self.latency}'
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
else:
urls = [url]

76
extras/scrapy_zsh_completion
View File

@ -14,40 +14,40 @@ _scrapy() {
;;
args)
case $words[1] in
bench)
(bench)
_scrapy_glb_opts
;;
fetch)
(fetch)
local options=(
'--headers[print response HTTP headers instead of body]'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
genspider)
(genspider)
local options=(
{-l,--list}'[List available templates]'
{-e,--edit}'[Edit spider after creating it]'
{'(--list)-l','(-l)--list'}'[List available templates]'
{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
'--force[If the spider already exists, overwrite it with the template]'
{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
'1:name:(NAME)'
'2:domain:_httpie_urls'
)
_scrapy_glb_opts $options
;;
runspider)
(runspider)
local options=(
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py'
)
_scrapy_glb_opts $options
;;
settings)
(settings)
local options=(
'--get=[print raw setting value]:option:(SETTING)'
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
@ -57,77 +57,77 @@ _scrapy() {
)
_scrapy_glb_opts $options
;;
shell)
(shell)
local options=(
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'::file:_files -g \*.html'
'::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
startproject)
(startproject)
local options=(
'1:name:(NAME)'
'2:dir:_dir_list'
)
_scrapy_glb_opts $options
;;
version)
(version)
local options=(
{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
)
_scrapy_glb_opts $options
;;
view)
(view)
local options=(
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
check)
(check)
local options=(
'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
{-v,--verbose}'[print contract tests for all spiders]'
{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
crawl)
(crawl)
local options=(
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
edit)
(edit)
local options=(
'1:spider:_scrapy_spiders'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
list)
(list)
_scrapy_glb_opts
;;
parse)
(parse)
local options=(
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
'--pipelines[process items through pipelines]'
"--nolinks[don't show links to follow (extracted requests)]"
"--noitems[don't show scraped items]"
'--nocolour[avoid using pygments to colorize the output]'
{-r,--rules}'[use CrawlSpider rules to discover the callback]'
{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{-v,--verbose}'[print each depth level one by one]'
{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
@ -162,7 +162,7 @@ _scrapy_cmds() {
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
commands=(${commands[@]} ${project_commands[@]})
fi
_describe -t common-commands 'common commands' commands
_describe -t common-commands 'common commands' commands && ret=0
}
_scrapy_glb_opts() {
@ -172,13 +172,13 @@ _scrapy_glb_opts() {
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
'--pidfile=[write process ID to FILE]:file:_files'
'--profile=[write python cProfile stats to FILE]:file:_files'
'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
'(-L --loglevel --logfile)--nolog[disable logging completely]'
'--pdb[enable pdb on failure]'
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
)
options=(${options[@]} "$@")
_arguments $options
_arguments -A "-*" $options && ret=0
}
_httpie_urls() {

122
pylintrc Normal file
View File

@ -0,0 +1,122 @@
[MASTER]
persistent=no
jobs=1 # >1 hides results
[MESSAGES CONTROL]
disable=abstract-method,
anomalous-backslash-in-string,
arguments-differ,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-continuation,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bad-whitespace,
bare-except,
blacklisted-name,
broad-except,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-dict-items,
consider-using-from-import,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
fixme,
function-redefined,
global-statement,
import-error,
import-outside-toplevel,
import-self,
inconsistent-return-statements,
inherit-non-class,
invalid-name,
invalid-overridden-method,
isinstance-second-argument-not-valid-type,
keyword-arg-before-vararg,
line-too-long,
logging-format-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
misplaced-comparison-constant,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-init,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-self-use,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-statement,
pointless-string-statement,
protected-access,
raise-missing-from,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-branches,
too-many-format-args,
too-many-function-args,
too-many-instance-attributes,
too-many-lines,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-lambda,
unnecessary-pass,
unreachable,
unspecified-encoding,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

234
pytest.ini
View File

@ -1,4 +1,5 @@
[pytest]
xfail_strict = true
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
@ -17,235 +18,8 @@ addopts =
--ignore=docs/topics/stats.rst
--ignore=docs/topics/telnetconsole.rst
--ignore=docs/utils
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-ignore =
W503
# Files that are only meant to provide top-level imports are expected not
# to use any of their imports:
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
# Issues pending a review:
# extras
extras/qps-bench-server.py E501
extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127 E502
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128 W504
scrapy/core/spidermw.py E501 E731 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E241
scrapy/core/downloader/webclient.py E731 E501 E128 E126
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E501
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
scrapy/extensions/closespider.py E501 E128 E123
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/throttle.py E501
# scrapy/http
scrapy/http/common.py E501
scrapy/http/cookies.py E501
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501 E731
# scrapy/loader
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/__init__.py E501
scrapy/pipelines/files.py E116 E501 E266
scrapy/pipelines/images.py E265 E501
scrapy/pipelines/media.py E125 E501 E266
# scrapy/selector
scrapy/selector/__init__.py F403
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
scrapy/settings/default_settings.py E501 E114 E116
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
scrapy/spidermiddlewares/referer.py E501 E129 W504
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/asyncio.py E501
scrapy/utils/benchserver.py E501
scrapy/utils/conf.py E402 E501
scrapy/utils/datatypes.py E501
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/gz.py E501 W504
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501
scrapy/utils/log.py E128 E501
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501
scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E501
scrapy/utils/reqser.py E501
scrapy/utils/request.py E127 E501
scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
scrapy/utils/spider.py E501
scrapy/utils/ssl.py E501
scrapy/utils/test.py E501
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/resolver.py E501
scrapy/responsetypes.py E128 E501
scrapy/robotstxt.py E501
scrapy/shell.py E501
scrapy/signalmanager.py E501
scrapy/spiderloader.py F841 E501 E126
scrapy/squeues.py E128
scrapy/statscollectors.py E501
# tests
tests/__init__.py E402 E501
tests/mockserver.py E401 E501 E126 E123
tests/pipelines.py F841
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E128 E127
tests/test_downloadermiddleware_retry.py E501 E128 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E128 E265
tests/test_item.py E128 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E501 E731 E741 E128 E117 E241
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128 E126
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E731 E128 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E241
tests/test_utils_log.py E741
tests/test_utils_python.py E501 E731
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
filterwarnings=
ignore::DeprecationWarning:twisted.web.test.test_webclient

2
scrapy/VERSION
View File

@ -1 +1 @@
1.8.0
2.5.0

47
scrapy/__init__.py
View File

@ -2,33 +2,11 @@
Scrapy - a web crawling and web scraping framework written for Python
"""
__all__ = ['__version__', 'version_info', 'twisted_version',
'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
# Scrapy version
import pkgutil
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
version_info = tuple(int(v) if v.isdigit() else v
for v in __version__.split('.'))
del pkgutil
# Check minimum required Python version
import sys
if sys.version_info < (3, 5):
print("Scrapy %s requires Python 3.5" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Declare top-level shortcuts
from scrapy.spiders import Spider
@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
from scrapy.selector import Selector
from scrapy.item import Item, Field
__all__ = [
'__version__', 'version_info', 'twisted_version', 'Spider',
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
]
# Scrapy and Twisted versions
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 6):
print(f"Scrapy {__version__} requires Python 3.6+")
sys.exit(1)
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del pkgutil
del sys
del warnings

11
scrapy/_monkeypatches.py
View File

@ -1,11 +0,0 @@
import copyreg
# Undo what Twisted's perspective broker adds to pickle register
# to prevent bugs like Twisted#7989 while serializing requests
import twisted.persisted.styles # NOQA
# Remove only entries with twisted serializers for non-twisted types.
for k, v in frozenset(copyreg.dispatch_table.items()):
if not str(getattr(k, '__module__', '')).startswith('twisted') \
and str(getattr(v, '__module__', '')).startswith('twisted'):
copyreg.dispatch_table.pop(k)

33
scrapy/cmdline.py
View File

@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
def _iter_command_classes(module_name):
@ -20,10 +19,12 @@ def _iter_command_classes(module_name):
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, ScrapyCommand) and \
obj.__module__ == module.__name__ and \
not obj == ScrapyCommand:
if (
inspect.isclass(obj)
and issubclass(obj, ScrapyCommand)
and obj.__module__ == module.__name__
and not obj == ScrapyCommand
):
yield obj
@ -43,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
else:
raise Exception("Invalid entry point %s" % entry_point.name)
raise Exception(f"Invalid entry point {entry_point.name}")
return cmds
@ -66,11 +67,11 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
@ -80,7 +81,7 @@ def _print_commands(settings, inproject):
print("Available commands:")
cmds = _get_commands_dict(settings, inproject)
for cmdname, cmdclass in sorted(cmds.items()):
print(" %-13s %s" % (cmdname, cmdclass.short_desc()))
print(f" {cmdname:<13} {cmdclass.short_desc()}")
if not inproject:
print()
print(" [ more ] More commands available when run from project directory")
@ -90,7 +91,7 @@ def _print_commands(settings, inproject):
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname)
print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
@ -118,7 +119,6 @@ def execute(argv=None, settings=None):
pass
else:
settings['EDITOR'] = editor
check_deprecated_settings(settings)
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
@ -133,7 +133,7 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
@ -155,7 +155,7 @@ def _run_command(cmd, args, opts):
def _run_command_profiled(cmd, args, opts):
if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)
@ -167,6 +167,7 @@ if __name__ == '__main__':
try:
execute()
finally:
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
# http://doc.pypy.org/en/latest/cpython_differences.html
# ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
garbage_collect()

58
scrapy/commands/__init__.py
View File

@ -3,19 +3,21 @@ Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
from typing import Any, Dict
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.exceptions import UsageError
class ScrapyCommand(object):
class ScrapyCommand:
requires_project = False
crawler_process = None
# default settings to be used for this command instead of global defaults
default_settings = {}
default_settings: Dict[str, Any] = {}
exitcode = 0
@ -23,7 +25,8 @@ class ScrapyCommand(object):
self.settings = None # set in scrapy.cmdline
def set_crawler(self, crawler):
assert not hasattr(self, '_crawler'), "crawler already set"
if hasattr(self, '_crawler'):
raise RuntimeError("crawler already set")
self._crawler = crawler
def syntax(self):
@ -40,14 +43,14 @@ class ScrapyCommand(object):
def long_desc(self):
"""A long description of the command. Return short description when not
available. It cannot contain newlines, since contents will be formatted
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
"""
return self.short_desc()
def help(self):
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines, since no post-formatting will
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
return self.long_desc()
@ -58,17 +61,17 @@ class ScrapyCommand(object):
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE",
help="write process ID to FILE")
help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)
@ -103,3 +106,34 @@ class ScrapyCommand(object):
Entry point for running commands
"""
raise NotImplementedError
class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output or opts.overwrite_output:
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')

4
scrapy/commands/bench.py
View File

@ -25,7 +25,7 @@ class Command(ScrapyCommand):
self.crawler_process.start()
class _BenchServer(object):
class _BenchServer:
def __enter__(self):
from scrapy.utils.test import get_testenv
@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

38
scrapy/commands/check.py
View File

@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult):
plural = "s" if run != 1 else ""
writeln(self.separator2)
writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s")
writeln()
infos = []
@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult):
write("FAILED")
failed, errored = map(len, (self.failures, self.errors))
if failed:
infos.append("failures=%d" % failed)
infos.append(f"failures={failed}")
if errored:
infos.append("errors=%d" % errored)
infos.append(f"errors={errored}")
else:
write("OK")
if infos:
writeln(" (%s)" % (", ".join(infos),))
writeln(f" ({', '.join(infos)})")
else:
write("\n")
@ -78,19 +78,19 @@ class Command(ScrapyCommand):
elif tested_methods:
self.crawler_process.crawl(spidercls)
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(' * %s' % method)
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(f' * {method}')
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())

47
scrapy/commands/crawl.py
View File

@ -1,11 +1,8 @@
import os
from scrapy.commands import ScrapyCommand
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.python import without_none_values
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = True
@ -15,43 +12,11 @@ class Command(ScrapyCommand):
def short_desc(self):
return "Run a spider"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
if opts.output == '-':
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(
self.settings.getwithbase('FEED_EXPORTERS'))
valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in valid_output_formats:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(valid_output_formats)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
@ -61,6 +26,8 @@ class Command(ScrapyCommand):
else:
self.crawler_process.start()
if self.crawler_process.bootstrap_failed or \
(hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
if (
self.crawler_process.bootstrap_failed
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
):
self.exitcode = 1

4
scrapy/commands/edit.py
View File

@ -32,8 +32,8 @@ class Command(ScrapyCommand):
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err("Spider not found: %s" % args[0])
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
self.exitcode = os.system('%s "%s"' % (editor, sfile))
self.exitcode = os.system(f'{editor} "{sfile}"')

14
scrapy/commands/fetch.py
View File

@ -19,16 +19,18 @@ class Command(ScrapyCommand):
return "Fetch a URL using the Scrapy downloader"
def long_desc(self):
return "Fetch a URL using the Scrapy downloader and print its content " \
"to stdout. You may want to use --nolog to disable logging"
return (
"Fetch a URL using the Scrapy downloader and print its content"
" to stdout. You may want to use --nolog to disable logging"
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
@ -49,8 +51,8 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
cb = lambda x: self._print_response(x, opts)
request = Request(args[0], callback=cb, dont_filter=True)
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:

90
scrapy/commands/genspider.py
View File

@ -4,6 +4,7 @@ import string
from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext
from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
@ -22,6 +23,14 @@ def sanitize_module_name(module_name):
return module_name
def extract_domain(url):
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == '' and o.netloc == '':
o = urlparse("//" + url.lstrip("/"))
return o.netloc
class Command(ScrapyCommand):
requires_project = False
@ -36,15 +45,15 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:
@ -59,39 +68,33 @@ class Command(ScrapyCommand):
if len(args) != 2:
raise UsageError()
name, domain = args[0:2]
name, url = args[0:2]
domain = extract_domain(url)
module = sanitize_module_name(name)
if self.settings.get('BOT_NAME') == module:
print("Cannot create a spider with the same name as your project")
return
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider already exists and not --force then halt
if not opts.force:
print("Spider %r already exists in module:" % name)
print(" %s" % spidercls.__module__)
return
if not opts.force and self._spider_exists(name):
return
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
if opts.edit:
self.exitcode = os.system('scrapy edit "%s"' % name)
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() \
for s in module.split('_'))
'classname': f'{capitalized_module}Spider'
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -99,29 +102,58 @@ class Command(ScrapyCommand):
else:
spiders_module = None
spiders_dir = "."
spider_file = "%s.py" % join(spiders_dir, module)
spider_file = f"{join(spiders_dir, module)}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r " % (name, \
template_name), end=('' if spiders_module else '\n'))
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, '%s.tmpl' % template)
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
return template_file
print("Unable to find template: %s\n" % template)
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(" %s" % splitext(filename)[0])
print(f" {splitext(filename)[0]}")
def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
return True
return False
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider with same name exists
print(f"Spider {name!r} already exists in module:")
print(f" {spidercls.__module__}")
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
return True
return False
@property
def templates_dir(self):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'spiders')
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'spiders'
)

67
scrapy/commands/parse.py
View File

@ -1,26 +1,26 @@
import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.commands import BaseRunSpiderCommand
from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils import display
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
items = {}
requests = {}
items: Dict[int, list] = {}
requests: Dict[int, list] = {}
first_response = None
@ -31,31 +31,29 @@ class Command(ScrapyCommand):
return "Parse URL (using its spider) and print the results"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
help="use this spider without looking for one")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
help="print each depth level one by one")
@property
def max_level(self):
@ -81,7 +79,7 @@ class Command(ScrapyCommand):
items = self.items.get(lvl, [])
print("# Scraped Items ", "-" * 60)
display.pprint([dict(x) for x in items], colorize=colour)
display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
def print_requests(self, lvl=None, colour=True):
if lvl is None:
@ -100,13 +98,13 @@ class Command(ScrapyCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print('\n>>> DEPTH LEVEL: %s <<<' % level)
print(f'\n>>> DEPTH LEVEL: {level} <<<')
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:
@ -117,7 +115,7 @@ class Command(ScrapyCommand):
items, requests = [], []
for x in iterate_spider_output(callback(response, **cb_kwargs)):
if isinstance(x, (BaseItem, dict)):
if is_item(x):
items.append(x)
elif isinstance(x, Request):
requests.append(x)
@ -146,9 +144,8 @@ class Command(ScrapyCommand):
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
@ -201,12 +198,15 @@ class Command(ScrapyCommand):
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
return requests
scraped_data += requests
return scraped_data
# update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
@ -222,18 +222,11 @@ class Command(ScrapyCommand):
return request
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
BaseRunSpiderCommand.process_options(self, args, opts)
self.process_spider_arguments(opts)
self.process_request_meta(opts)
self.process_request_cb_kwargs(opts)
def process_spider_arguments(self, opts):
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_request_meta(self, opts):
if opts.meta:
try:

46
scrapy/commands/runspider.py
View File

@ -3,18 +3,16 @@ import os
from importlib import import_module
from scrapy.utils.spider import iter_spider_classes
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.python import without_none_values
from scrapy.commands import BaseRunSpiderCommand
def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext != '.py':
raise ValueError("Not a Python source file: %s" % abspath)
if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
@ -25,7 +23,7 @@ def _import_file(filepath):
return module
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
@ -39,49 +37,19 @@ class Command(ScrapyCommand):
def long_desc(self):
return "Run the spider defined in the given file"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
if opts.output == '-':
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in feed_exporters:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(feed_exporters)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
raise UsageError("File not found: %s\n" % filename)
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError("Unable to load %r: %s\n" % (filename, e))
raise UsageError(f"Unable to load {filename!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError("No spider found in file: %s\n" % filename)
raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop()
self.crawler_process.crawl(spidercls, **opts.spargs)

10
scrapy/commands/settings.py
View File

@ -19,15 +19,15 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings

Some files were not shown because too many files have changed in this diff Show More