RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into auth-creds-from-url

This commit is contained in:
Adrián Chaves 2020-04-06 02:21:07 +02:00
parent 9fc45cccb5 8845773d44
commit fa4b13ad03
512 changed files with 21903 additions and 14415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.bandit.yml Normal file
View File

@ -0,0 +1,18 @@
skips:
- B101
- B105
- B301
- B303
- B306
- B307
- B311
- B320
- B321
- B402 # https://github.com/scrapy/scrapy/issues/4180
- B403
- B404
- B406
- B410
- B503
- B603
- B605

24
.bumpversion.cfg
View File

@ -1,29 +1,7 @@
[bumpversion]
current_version = 1.2.1
current_version = 2.0.0
commit = True
tag = True
tag_name = {new_version}
parse = ^
(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
(?:(?P<prerel>[abc]|rc|dev)(?P<prerelversion>\d+))?
serialize =
{major}.{minor}.{patch}{prerel}{prerelversion}
{major}.{minor}.{patch}
[bumpversion:file:scrapy/VERSION]
[bumpversion:part:prerel]
optional_value = gamma
values =
dev
rc
gamma
[bumpversion:part:prerelversion]
values =
1
2
3
4
5

14
.coveragerc
View File

@ -3,17 +3,3 @@ branch = true
include = scrapy/*
omit =
tests/*
scrapy/xlib/*
scrapy/conf.py
scrapy/stats.py
scrapy/project.py
scrapy/utils/decorator.py
scrapy/statscol.py
scrapy/squeue.py
scrapy/log.py
scrapy/dupefilter.py
scrapy/command.py
scrapy/linkextractor.py
scrapy/spider.py
scrapy/contrib/*
scrapy/contrib_exp/*

41
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@ -0,0 +1,41 @@
---
name: Bug report
about: Report a problem to help us improve
---
<!--
Thanks for taking an interest in Scrapy!
If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
The following is a suggested template to structure your issue, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#reporting-bugs
-->
### Description
[Description of the issue]
### Steps to Reproduce
1. [First Step]
2. [Second Step]
3. [and so on...]
**Expected behavior:** [What you expect to happen]
**Actual behavior:** [What actually happens]
**Reproduces how often:** [What percentage of the time does it reproduce?]
### Versions
Please paste here the output of executing `scrapy version --verbose` in the command line.
### Additional context
Any additional information, configuration, data or output from commands that might be necessary to reproduce or understand the issue. Please try not to include screenshots of code or the command line, paste the contents as text instead. You can use [GitHub Flavored Markdown](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) to make the text look better.

33
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View File

@ -0,0 +1,33 @@
---
name: Feature request
about: Suggest an idea for an enhancement or new feature
---
<!--
Thanks for taking an interest in Scrapy!
If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
The following is a suggested template to structure your pull request, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#writing-patches and https://doc.scrapy.org/en/latest/contributing.html#submitting-patches
-->
## Summary
One paragraph explanation of the feature.
## Motivation
Why are we doing this? What use cases does it support? What is the expected outcome?
## Describe alternatives you've considered
A clear and concise description of the alternative solutions you've considered. Be sure to explain why Scrapy's existing customizability isn't suitable for this feature.
## Additional context
Any additional information about the feature request here.

3
.gitignore vendored
View File

@ -12,6 +12,9 @@ dist
.idea
htmlcov/
.coverage
.pytest_cache/
.coverage.*
.cache/
# Windows
Thumbs.db

11
.readthedocs.yml Normal file
View File

@ -0,0 +1,11 @@
version: 2
sphinx:
configuration: docs/conf.py
fail_on_warning: true
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.7 # Keep in sync with .travis.yml
install:
- requirements: docs/requirements.txt
- path: .

49
.travis.yml
View File

@ -1,19 +1,46 @@
language: python
python: 3.5
sudo: false
dist: xenial
branches:
only:
- master
- /^\d\.\d+$/
- /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
env:
- TOXENV=py27
- TOXENV=jessie
- TOXENV=py33
- TOXENV=py35
- TOXENV=docs
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
- env: TOXENV=security
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
- env: TOXENV=py35
python: 3.5
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py35-asyncio
python: 3.5.2
- env: TOXENV=py36
python: 3.6
- env: TOXENV=py37
python: 3.7
- env: TOXENV=py38
python: 3.8
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=py38-asyncio
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
install:
- pip install -U tox twine wheel codecov
- |
if [ "$TOXENV" = "pypy3" ]; then
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
- pip install -U tox twine wheel codecov
script: tox
after_success:
- codecov
@ -35,4 +62,4 @@ deploy:
on:
tags: true
repo: scrapy/scrapy
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$"
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

80
CODE_OF_CONDUCT.md
View File

@ -1,24 +1,41 @@
# Contributor Code of Conduct
# Contributor Covenant Code of Conduct
As contributors and maintainers of this project, and in the interest of
fostering an open and welcoming community, we pledge to respect all people who
contribute through reporting issues, posting feature requests, updating
documentation, submitting pull requests or patches, and other activities.
## Our Pledge
We are committed to making participation in this project a harassment-free
experience for everyone, regardless of level of experience, gender, gender
identity and expression, sexual orientation, disability, personal appearance,
body size, race, ethnicity, age, religion, or nationality.
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery
* Personal attacks
* Trolling or insulting/derogatory comments
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing other's private information, such as physical or electronic
addresses, without explicit permission
* Other unethical or unprofessional conduct
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
@ -26,25 +43,32 @@ that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
By adopting this Code of Conduct, project maintainers commit themselves to
fairly and consistently applying these principles to every aspect of managing
this project. Project maintainers who do not follow or enforce the Code of
Conduct may be permanently removed from the project team.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community.
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting a project maintainer at opensource@scrapinghub.com. All
reported by contacting the project team at opensource@scrapinghub.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. Maintainers are
obligated to maintain confidentiality with regard to the reporter of an
incident.
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 1.3.0, available at
[http://contributor-covenant.org/version/1/3/0/][version]
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/3/0/
[version]: http://contributor-covenant.org/version/1/4/

4
CONTRIBUTING.md
View File

@ -1,6 +1,6 @@
The guidelines for contributing are available here:
http://doc.scrapy.org/en/master/contributing.html
https://docs.scrapy.org/en/master/contributing.html
Please do not abuse the issue tracker for support questions.
If your issue topic can be rephrased to "How to ...?", please use the
support channels to get it answered: http://scrapy.org/community/
support channels to get it answered: https://scrapy.org/community/

2
INSTALL
View File

@ -1,4 +1,4 @@
For information about installing Scrapy see:
* docs/intro/install.rst (local file)
* http://doc.scrapy.org/en/latest/intro/install.html (online version)
* https://docs.scrapy.org/en/latest/intro/install.html (online version)

10
LICENSE
View File

@ -4,11 +4,11 @@ All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
1. Redistributions of source code must retain the above copyright notice,
this list of conditions, and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions, and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used

12
MANIFEST.in
View File

@ -3,12 +3,24 @@ include AUTHORS
include INSTALL
include LICENSE
include MANIFEST.in
include NEWS
include scrapy/VERSION
include scrapy/mime.types
include codecov.yml
include conftest.py
include pytest.ini
include requirements-*.txt
include tox.ini
recursive-include scrapy/templates *
recursive-include scrapy license.txt
recursive-include docs *
prune docs/build
recursive-include extras *
recursive-include bin *
recursive-include tests *
global-exclude __pycache__ *.py[cod]

24
Makefile.buildbot
View File

@ -1,24 +0,0 @@
TRIAL := $(shell which trial)
BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
export PYTHONPATH=$(PWD)
test:
coverage run --branch $(TRIAL) --reporter=text tests
rm -rf htmlcov && coverage html
-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
build:
git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION
debchange -m -D unstable --force-distribution -v \
$$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \
"Automatic build"
debuild -us -uc -b
clean:
git checkout debian scrapy/VERSION
git clean -dfq
pypi:
umask 0022 && chmod -R a+rX . && python setup.py sdist upload
.PHONY: clean test build

50
README.rst
View File

@ -6,20 +6,20 @@ Scrapy
:target: https://pypi.python.org/pypi/Scrapy
:alt: PyPI Version
.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg
:target: https://pypi.python.org/pypi/Scrapy
:alt: Supported Python Versions
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
:target: http://travis-ci.org/scrapy/scrapy
:target: https://travis-ci.org/scrapy/scrapy
:alt: Build Status
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
:target: https://pypi.python.org/pypi/Scrapy
:alt: Wheel Status
.. image:: http://static.scrapy.org/py3progress/badge.svg
:target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting
:alt: Python 3 Porting Status
.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
:target: http://codecov.io/github/scrapy/scrapy?branch=master
:target: https://codecov.io/github/scrapy/scrapy?branch=master
:alt: Coverage report
.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg
@ -34,14 +34,14 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
For more information including a list of features check the Scrapy homepage at:
http://scrapy.org
Check the Scrapy homepage at https://scrapy.org for more information,
including a list of features.
Requirements
============
* Python 2.7 or Python 3.3+
* Works on Linux, Windows, Mac OSX, BSD
* Python 3.5+
* Works on Linux, Windows, macOS, BSD
Install
=======
@ -50,43 +50,45 @@ The quick way::
pip install scrapy
For more details see the install section in the documentation:
http://doc.scrapy.org/en/latest/intro/install.html
Releases
========
You can download the latest stable and development releases from:
http://scrapy.org/download/
See the install section in the documentation at
https://docs.scrapy.org/en/latest/intro/install.html for more details.
Documentation
=============
Documentation is available online at http://doc.scrapy.org/ and in the ``docs``
Documentation is available online at https://docs.scrapy.org/ and in the ``docs``
directory.
Releases
========
You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
Community (blog, twitter, mail list, IRC)
=========================================
See http://scrapy.org/community/
See https://scrapy.org/community/ for details.
Contributing
============
See https://docs.scrapy.org/en/master/contributing.html for details.
Code of Conduct
---------------
Please note that this project is released with a Contributor Code of Conduct
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
By participating in this project you agree to abide by its terms.
Please report unacceptable behavior to opensource@scrapinghub.com.
See http://doc.scrapy.org/en/master/contributing.html
Companies using Scrapy
======================
See http://scrapy.org/companies/
See https://scrapy.org/companies/ for a list.
Commercial Support
==================
See http://scrapy.org/support/
See https://scrapy.org/support/ for details.

25
appveyor.yml Normal file
View File

@ -0,0 +1,25 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
- "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

5
artwork/README → artwork/README.rst
View File

@ -1,3 +1,4 @@
==============
Scrapy artwork
==============
@ -8,10 +9,10 @@ scrapy-logo.jpg
Main Scrapy logo, in JPEG format.
qlassik.zip
qlassik.zip
-----------
Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font
Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
scrapy-blog.logo.xcf
--------------------

6
codecov.yml Normal file
View File

@ -0,0 +1,6 @@
comment:
layout: "header, diff, tree"
coverage:
status:
project: false

65
conftest.py
View File

@ -1,44 +1,53 @@
import glob
import six
from pathlib import Path
import pytest
from twisted import version as twisted_version
def _py_files(folder):
return glob.glob(folder + "/*.py") + glob.glob(folder + "/*/*.py")
return (str(p) for p in Path(folder).rglob('*.py'))
collect_ignore = [
# deprecated or moved modules
"scrapy/conf.py",
"scrapy/stats.py",
"scrapy/project.py",
"scrapy/utils/decorator.py",
"scrapy/statscol.py",
"scrapy/squeue.py",
"scrapy/log.py",
"scrapy/dupefilter.py",
"scrapy/command.py",
"scrapy/linkextractor.py",
"scrapy/spider.py",
# not a test, but looks like a test
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]
] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
collect_ignore += _py_files("scrapy/xlib/tx")
if six.PY3:
for line in open('tests/py3-ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
@pytest.fixture()
def chdir(tmpdir):
"""Change to pytest-provided temporary directory"""
tmpdir.chdir()
def pytest_collection_modifyitems(session, config, items):
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
try:
from pytest_flake8 import Flake8Item
if config.getoption('--flake8'):
items[:] = [item for item in items if isinstance(item, Flake8Item)]
except ImportError:
pass
@pytest.fixture(scope='class')
def reactor_pytest(request):
if not request.cls:
# doctests
return
request.cls.reactor_pytest = request.config.getoption("--reactor")
return request.cls.reactor_pytest
@pytest.fixture(autouse=True)
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')

5
debian/changelog vendored
View File

@ -1,5 +0,0 @@
scrapy (0.11) unstable; urgency=low
* Initial release.
-- Scrapinghub Team <info@scrapinghub.com> Thu, 10 Jun 2010 17:24:02 -0300

1
debian/compat vendored
View File

@ -1 +0,0 @@
7

20
debian/control vendored
View File

@ -1,20 +0,0 @@
Source: scrapy
Section: python
Priority: optional
Maintainer: Scrapinghub Team <info@scrapinghub.com>
Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
Standards-Version: 3.8.4
Homepage: http://scrapy.org/
Package: scrapy
Architecture: all
Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
Recommends: python-setuptools
Conflicts: python-scrapy, scrapy-0.25
Provides: python-scrapy, scrapy-0.25
Description: Python web crawling and web scraping framework
Scrapy is a fast high-level web crawling and web scraping framework,
used to crawl websites and extract structured data from their pages.
It can be used for a wide range of purposes, from data mining to
monitoring and automated testing.

40
debian/copyright vendored
View File

@ -1,40 +0,0 @@
This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
It was downloaded from http://scrapy.org
Upstream Author: Scrapy Developers
Copyright: 2007-2013 Scrapy Developers
License: bsd
Copyright (c) Scrapy developers.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The Debian packaging is (C) 2010-2013, Scrapinghub <info@scrapinghub.com> and
is licensed under the BSD, see `/usr/share/common-licenses/BSD'.

1
debian/pyversions vendored
View File

@ -1 +0,0 @@
2.7

5
debian/rules vendored
View File

@ -1,5 +0,0 @@
#!/usr/bin/make -f
# -*- makefile -*-
%:
dh $@

2
debian/scrapy.docs vendored
View File

@ -1,2 +0,0 @@
README.rst
AUTHORS

2
debian/scrapy.install vendored
View File

@ -1,2 +0,0 @@
extras/scrapy_bash_completion etc/bash_completion.d/
extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy

2
debian/scrapy.lintian-overrides vendored
View File

@ -1,2 +0,0 @@
new-package-should-close-itp-bug
extra-license-file usr/share/pyshared/scrapy/xlib/pydispatch/license.txt

1
debian/scrapy.manpages vendored
View File

@ -1 +0,0 @@
extras/scrapy.1

9
docs/Makefile
View File

@ -10,7 +10,8 @@ PAPER =
SOURCES =
SHELL = /bin/bash
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
-D latex_elements.papersize=$(PAPER) \
$(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
.PHONY: help update build html htmlhelp clean
@ -81,8 +82,12 @@ pydoc-topics: build
@echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \
"into the Lib/ directory"
coverage: BUILDER = coverage
coverage: build
htmlview: html
$(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')"
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
clean:
-rm -rf build/*

6
docs/README → docs/README.rst
View File

@ -1,3 +1,5 @@
:orphan:
======================================
Scrapy documentation quick start guide
======================================
@ -9,11 +11,11 @@ Setup the environment
---------------------
To compile the documentation you need Sphinx Python library. To install it
and all its dependencies run
and all its dependencies run the following command from this dir
::
pip install 'Sphinx >= 1.3'
pip install -r requirements.txt
Compile the documentation

6
docs/_ext/scrapydocs.py
View File

@ -1,6 +1,6 @@
from docutils.parsers.rst.roles import set_classes
from docutils import nodes
from sphinx.util.compat import Directive
from docutils.parsers.rst import Directive
from sphinx.util.nodes import make_refnode
from operator import itemgetter
@ -110,24 +110,28 @@ def setup(app):
app.connect('doctree-read', collect_scrapy_settings_refs)
app.connect('doctree-resolved', replace_settingslist_nodes)
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
set_classes(options)
node = nodes.reference(rawtext, text, refuri=ref, **options)
return [node], []
def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/issues/' + text
set_classes(options)
node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
return [node], []
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'https://github.com/scrapy/scrapy/commit/' + text
set_classes(options)
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
return [node], []
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
set_classes(options)

281
docs/_tests/quotes.html Normal file
View File

@ -0,0 +1,281 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Quotes to Scrape</title>
<link rel="stylesheet" href="/static/bootstrap.min.css">
<link rel="stylesheet" href="/static/main.css">
</head>
<body>
<div class="container">
<div class="row header-box">
<div class="col-md-8">
<h1>
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
</h1>
</div>
<div class="col-md-4">
<p>
<a href="/login">Login</a>
</p>
</div>
</div>
<div class="row">
<div class="col-md-8">
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
<a class="tag" href="/tag/change/page/1/">change</a>
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
<a class="tag" href="/tag/world/page/1/">world</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
<a href="/author/J-K-Rowling">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
<a class="tag" href="/tag/choices/page/1/">choices</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/live/page/1/">live</a>
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
<span>by <small class="author" itemprop="author">Jane Austen</small>
<a href="/author/Jane-Austen">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
<a class="tag" href="/tag/books/page/1/">books</a>
<a class="tag" href="/tag/classic/page/1/">classic</a>
<a class="tag" href="/tag/humor/page/1/">humor</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
<a href="/author/Marilyn-Monroe">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
<a class="tag" href="/tag/success/page/1/">success</a>
<a class="tag" href="/tag/value/page/1/">value</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
<span>by <small class="author" itemprop="author">André Gide</small>
<a href="/author/Andre-Gide">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="life,love" / >
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/love/page/1/">love</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
<a href="/author/Thomas-A-Edison">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
<a class="tag" href="/tag/edison/page/1/">edison</a>
<a class="tag" href="/tag/failure/page/1/">failure</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
<a href="/author/Eleanor-Roosevelt">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
<span>by <small class="author" itemprop="author">Steve Martin</small>
<a href="/author/Steve-Martin">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
<a class="tag" href="/tag/humor/page/1/">humor</a>
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
<a class="tag" href="/tag/simile/page/1/">simile</a>
</div>
</div>
<nav>
<ul class="pager">
<li class="next">
<a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
</li>
</ul>
</nav>
</div>
<div class="col-md-4 tags-box">
<h2>Top Ten tags</h2>
<span class="tag-item">
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
</span>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="text-muted">
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
</p>
</div>
</footer>
</body>
</html>

281
docs/_tests/quotes1.html Normal file
View File

@ -0,0 +1,281 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Quotes to Scrape</title>
<link rel="stylesheet" href="/static/bootstrap.min.css">
<link rel="stylesheet" href="/static/main.css">
</head>
<body>
<div class="container">
<div class="row header-box">
<div class="col-md-8">
<h1>
<a href="/" style="text-decoration: none">Quotes to Scrape</a>
</h1>
</div>
<div class="col-md-4">
<p>
<a href="/login">Login</a>
</p>
</div>
</div>
<div class="row">
<div class="col-md-8">
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" / >
<a class="tag" href="/tag/change/page/1/">change</a>
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
<a class="tag" href="/tag/world/page/1/">world</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
<span>by <small class="author" itemprop="author">J.K. Rowling</small>
<a href="/author/J-K-Rowling">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="abilities,choices" / >
<a class="tag" href="/tag/abilities/page/1/">abilities</a>
<a class="tag" href="/tag/choices/page/1/">choices</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" / >
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/live/page/1/">live</a>
<a class="tag" href="/tag/miracle/page/1/">miracle</a>
<a class="tag" href="/tag/miracles/page/1/">miracles</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
<span>by <small class="author" itemprop="author">Jane Austen</small>
<a href="/author/Jane-Austen">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" / >
<a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
<a class="tag" href="/tag/books/page/1/">books</a>
<a class="tag" href="/tag/classic/page/1/">classic</a>
<a class="tag" href="/tag/humor/page/1/">humor</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
<span>by <small class="author" itemprop="author">Marilyn Monroe</small>
<a href="/author/Marilyn-Monroe">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" / >
<a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
<span>by <small class="author" itemprop="author">Albert Einstein</small>
<a href="/author/Albert-Einstein">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="adulthood,success,value" / >
<a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
<a class="tag" href="/tag/success/page/1/">success</a>
<a class="tag" href="/tag/value/page/1/">value</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
<span>by <small class="author" itemprop="author">André Gide</small>
<a href="/author/Andre-Gide">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="life,love" / >
<a class="tag" href="/tag/life/page/1/">life</a>
<a class="tag" href="/tag/love/page/1/">love</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
<span>by <small class="author" itemprop="author">Thomas A. Edison</small>
<a href="/author/Thomas-A-Edison">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" / >
<a class="tag" href="/tag/edison/page/1/">edison</a>
<a class="tag" href="/tag/failure/page/1/">failure</a>
<a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
<a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
<span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
<a href="/author/Eleanor-Roosevelt">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" / >
<a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
</div>
</div>
<div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
<span>by <small class="author" itemprop="author">Steve Martin</small>
<a href="/author/Steve-Martin">(about)</a>
</span>
<div class="tags">
Tags:
<meta class="keywords" itemprop="keywords" content="humor,obvious,simile" / >
<a class="tag" href="/tag/humor/page/1/">humor</a>
<a class="tag" href="/tag/obvious/page/1/">obvious</a>
<a class="tag" href="/tag/simile/page/1/">simile</a>
</div>
</div>
<nav>
<ul class="pager">
<li class="next">
<a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
</li>
</ul>
</nav>
</div>
<div class="col-md-4 tags-box">
<h2>Top Ten tags</h2>
<span class="tag-item">
<a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
</span>
<span class="tag-item">
<a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
</span>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="text-muted">
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
</p>
</div>
</footer>
</body>
</html>

91
docs/conf.py
View File

@ -12,6 +12,7 @@
# serve to show the default.
import sys
from datetime import datetime
from os import path
# If your extensions are in another directory, add it here. If the directory
@ -27,8 +28,13 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'hoverxref.extension',
'notfound.extension',
'scrapydocs',
'sphinx.ext.autodoc'
'sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
@ -44,8 +50,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'Scrapy'
copyright = u'2008-2016, Scrapy developers'
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -73,6 +79,8 @@ language = 'en'
# List of documents that shouldn't be included in the build.
#unused_docs = []
exclude_patterns = ['build']
# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['.build']
@ -144,10 +152,6 @@ html_static_path = ['_static']
# using the given strftime format.
html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
@ -191,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', ur'Scrapy Documentation',
ur'Scrapy developers', 'manual'),
('index', 'Scrapy.tex', 'Scrapy Documentation',
'Scrapy developers', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@ -222,3 +226,72 @@ linkcheck_ignore = [
'http://localhost:\d+', 'http://hg.scrapy.org',
'http://directory.google.com/'
]
# Options for the Coverage extension
# ----------------------------------
coverage_ignore_pyobjects = [
# Contract’s add_pre_hook and add_post_hook are not documented because
# they should be transparent to contract developers, for whom pre_hook and
# post_hook should be the actual concern.
r'\bContract\.add_(pre|post)_hook$',
# ContractsManager is an internal class, developers are not expected to
# interact with it directly in any way.
r'\bContractsManager\b$',
# For default contracts we only want to document their general purpose in
# their __init__ method, the methods they reimplement to achieve that purpose
# should be irrelevant to developers using those contracts.
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
# Methods of downloader middlewares are not documented, only the classes
# themselves, since downloader middlewares are controlled through Scrapy
# settings.
r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
# Base classes of downloader middlewares are implementation details that
# are not meant for users.
r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
# Private exception used by the command-line interface implementation.
r'^scrapy\.exceptions\.UsageError',
# Methods of BaseItemExporter subclasses are only documented in
# BaseItemExporter.
r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
# Extension behavior is only modified through settings. Methods of
# extension classes, as well as helper functions, are implementation
# details that are not documented.
r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.item\.DictItem$',
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
# Implementation detail of LxmlLinkExtractor
r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
]
# Options for the InterSphinx extension
# -------------------------------------
intersphinx_mapping = {
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
}
# Options for sphinx-hoverxref options
# ------------------------------------
hoverxref_auto_ref = True

29
docs/conftest.py Normal file
View File

@ -0,0 +1,29 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.codeblock import CodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
def load_response(url, filename):
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
with open(input_path, 'rb') as input_file:
return HtmlResponse(url, body=input_file.read())
def setup(namespace):
namespace['load_response'] = load_response
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
CodeBlockParser(future_imports=['print_function']),
skip,
],
pattern='*.rst',
setup=setup,
).pytest()

175
docs/contributing.rst
View File

@ -6,25 +6,29 @@ Contributing to Scrapy
.. important::
Double check you are reading the most recent version of this document at
http://doc.scrapy.org/en/master/contributing.html
Double check that you are reading the most recent version of this document at
https://docs.scrapy.org/en/master/contributing.html
There are many ways to contribute to Scrapy. Here are some of them:
* Blog about Scrapy. Tell the world how you're using Scrapy. This will help
newcomers with more examples and the Scrapy project to increase its
newcomers with more examples and will help the Scrapy project to increase its
visibility.
* Report bugs and request features in the `issue tracker`_, trying to follow
the guidelines detailed in `Reporting bugs`_ below.
* Submit patches for new functionality and/or bug fixes. Please read
`Writing patches`_ and `Submitting patches`_ below for details on how to
* Submit patches for new functionalities and/or bug fixes. Please read
:ref:`writing-patches` and `Submitting patches`_ below for details on how to
write and submit a patch.
* Join the `scrapy-users`_ mailing list and share your ideas on how to
* Join the `Scrapy subreddit`_ and share your ideas on how to
improve Scrapy. We're always open to suggestions.
* Answer Scrapy questions at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__.
Reporting bugs
==============
@ -35,18 +39,23 @@ Reporting bugs
trusted Scrapy developers, and its archives are not public.
Well-written bug reports are very helpful, so keep in mind the following
guidelines when reporting a new bug.
guidelines when you're going to report a new bug.
* check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
well-known question
* check the `open issues`_ to see if it has already been reported. If it has,
don't dismiss the report but check the ticket history and comments, you may
find additional useful information to contribute.
* if you have a general question about Scrapy usage, please ask it at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
(use "scrapy" tag).
* search the `scrapy-users`_ list to see if it has been discussed there, or
if you're not sure if what you're seeing is a bug. You can also ask in the
`#scrapy` IRC channel.
* check the `open issues`_ to see if the issue has already been reported. If it
has, don't dismiss the report, but check the ticket history and comments. If
you have additional useful information, please leave a comment, or consider
:ref:`sending a pull request <writing-patches>` with a fix.
* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
been discussed there, or if you're not sure if what you're seeing is a bug.
You can also ask in the ``#scrapy`` IRC channel.
* write **complete, reproducible, specific bug reports**. The smaller the test
case, the better. Remember that other developers won't have your project to
@ -54,17 +63,24 @@ guidelines when reporting a new bug.
it. See for example StackOverflow's guide on creating a
`Minimal, Complete, and Verifiable example`_ exhibiting the issue.
* the most awesome way to provide a complete reproducible example is to
send a pull request which adds a failing test case to the
Scrapy testing suite (see :ref:`submitting-patches`).
This is helpful even if you don't have an intention to
fix the issue yourselves.
* include the output of ``scrapy version -v`` so developers working on your bug
know exactly which version and platform it occurred on, which is often very
helpful for reproducing it, or knowing if it was already fixed.
.. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve
.. _writing-patches:
Writing patches
===============
The better written a patch is, the higher chance that it'll get accepted and
the sooner that will be merged.
The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged.
Well-written patches should:
@ -83,6 +99,17 @@ Well-written patches should:
the documentation changes in the same patch. See `Documentation policies`_
below.
* if you're adding a private API, please add a regular expression to the
``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new
private API from documentation coverage checks.
To see if your private API is skipped properly, generate a documentation
coverage report as follows::
tox -e docs-coverage
.. _submitting-patches:
Submitting patches
==================
@ -98,16 +125,34 @@ patch, but it's always good to have a patch ready to illustrate your arguments
and show that you have put some additional thought into the subject. A good
starting point is to send a pull request on GitHub. It can be simple enough to
illustrate your idea, and leave documentation/tests for later, after the idea
has been validated and proven useful. Alternatively, you can send an email to
`scrapy-users`_ to discuss your idea first.
has been validated and proven useful. Alternatively, you can start a
conversation in the `Scrapy subreddit`_ to discuss your idea first.
Sometimes there is an existing pull request for the problem you'd like to
solve, which is stalled for some reason. Often the pull request is in a
right direction, but changes are requested by Scrapy maintainers, and the
original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch
by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
(replace 'upstream' with a remote name for scrapy repository,
``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
with a name of the branch you want to create locally).
See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally.
When writing GitHub pull requests, try to keep titles short but descriptive.
E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
prefer "Fix hanging when exception occurs in start_requests (#411)"
instead of "Fix for #411".
Complete titles make it easy to skim through the issue tracker.
instead of "Fix for #411". Complete titles make it easy to skim through
the issue tracker.
Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
removal, etc) in separate commits than functional changes. This will make pull
removal, etc) in separate commits from functional changes. This will make pull
requests easier to review and more likely to get merged.
Coding style
@ -121,63 +166,82 @@ Scrapy:
* It's OK to use lines longer than 80 chars if it improves the code
readability.
* Don't put your name in the code you contribute. Our policy is to keep
the contributor's name in the `AUTHORS`_ file distributed with Scrapy.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
Scrapy Contrib
==============
Scrapy contrib shares a similar rationale as Django contrib, which is explained
in `this post <https://jacobian.org/writing/what-is-django-contrib/>`_. If you
are working on a new functionality, please follow that rationale to decide
whether it should be a Scrapy contrib. If unsure, you can ask in
`scrapy-users`_.
.. _documentation-policies:
Documentation policies
======================
* **Don't** use docstrings for documenting classes, or methods which are
already documented in the official (sphinx) documentation. For example, the
:meth:`ItemLoader.add_value` method should be documented in the sphinx
documentation, not its docstring.
For reference documentation of API members (classes, methods, etc.) use
docstrings and make sure that the Sphinx documentation uses the
:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
documentation should follow docstring conventions (`PEP 257`_) and be
IDE-friendly: short, to the point, and it may provide short examples.
* **Do** use docstrings for documenting functions not present in the official
(sphinx) documentation, such as functions from ``scrapy.utils`` package and
its sub-modules.
Other types of documentation, such as tutorials or topics, should be covered in
files within the ``docs/`` directory. This includes documentation that is
specific to an API member, but goes beyond API reference documentation.
In any case, if something is covered in a docstring, use the
:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
documentation instead of duplicating the docstring in files within the
``docs/`` directory.
Tests
=====
Tests are implemented using the `Twisted unit-testing framework`_, running
tests requires `tox`_.
Tests are implemented using the :doc:`Twisted unit-testing framework
<twisted:core/development/policy/test-standard>`. Running tests requires
:doc:`tox <tox:index>`.
.. _running-tests:
Running tests
-------------
Make sure you have a recent enough `tox`_ installation:
To run all tests::
``tox --version``
If your version is older than 1.7.0, please update it first:
``pip install -U tox``
To run all tests go to the root directory of Scrapy source code and run:
``tox``
tox
To run a specific test (say ``tests/test_loader.py``) use:
``tox -- tests/test_loader.py``
To see coverage report install `coverage`_ (``pip install coverage``) and run:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
the tests with Python 3.6 use::
tox -e py36
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py36,py38 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
default positional arguments defined in ``tox.ini``, so you must include those
default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
tox -e py36 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:
``coverage report``
see output of ``coverage --help`` for more options like html or xml report.
.. _coverage: https://pypi.python.org/pypi/coverage
Writing tests
-------------
@ -197,9 +261,10 @@ And their unit-tests are in::
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
.. _Scrapy subreddit: https://reddit.com/r/scrapy
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
.. _open issues: https://github.com/scrapy/scrapy/issues
.. _pull request: https://help.github.com/send-pull-requests/
.. _tox: https://pypi.python.org/pypi/tox
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist

74
docs/faq.rst
View File

@ -21,9 +21,9 @@ Python code.
In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
comparing `jinja2`_ to `Django`_.
.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
.. _jinja2: http://jinja.pocoo.org/
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: https://lxml.de/
.. _jinja2: https://palletsprojects.com/p/jinja/
.. _Django: https://www.djangoproject.com/
Can I use Scrapy with BeautifulSoup?
@ -69,12 +69,15 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
Scrapy is supported under Python 2.7 and Python 3.3+.
Python 2.6 support was dropped starting at Scrapy 0.20.
Scrapy is supported under Python 3.5+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
Python 2 support was dropped in Scrapy 2.0.
.. note::
Python 3 is not yet supported on Windows.
For Python 3 support on Windows, it is recommended to use
Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
Did Scrapy "steal" X from Django?
---------------------------------
@ -126,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order?
By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
basically means that it crawls in `DFO order`_. This order is more convenient
in most cases. If you do want to crawl in true `BFO order`_, you can do it by
in most cases.
If you do want to crawl in true `BFO order`_, you can do it by
setting the following settings::
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
While pending requests are below the configured values of
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
concurrently. As a result, the first few requests of a crawl rarely follow the
desired order. Lowering those settings to ``1`` enforces the desired order, but
it significantly slows down the crawl as a whole.
My Scrapy crawler has memory leaks. What can I do?
--------------------------------------------------
@ -256,7 +269,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For
more info on how it works see `this page`_. Also, here's an `example spider`_
which scrapes one of these sites.
.. _this page: http://search.cpan.org/~ecarroll/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
.. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py
What's the best way to parse big XML/CSV data feeds?
@ -316,7 +329,50 @@ I'm scraping a XML document and my XPath selector doesn't return any items
You may need to remove namespaces. See :ref:`removing-namespaces`.
.. _faq-split-item:
How to split an item into multiple items in an item pipeline?
-------------------------------------------------------------
:ref:`Item pipelines <topics-item-pipeline>` cannot yield multiple items per
input item. :ref:`Create a spider middleware <custom-spider-middleware>`
instead, and use its
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method for this purpose. For example::
from copy import deepcopy
from scrapy.item import BaseItem
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if isinstance(item, (BaseItem, dict)):
for _ in range(item['multiply_by']):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?
-----------------------------------
Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
Note that by doing so, you lose the ability to set a specific timeout for DNS requests
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
.. _faq-specific-reactor:
How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?
----------------------------------------------------------------------------------------------
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/LIFO
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search

44
docs/index.rst
View File

@ -4,7 +4,13 @@
Scrapy |version| documentation
==============================
This documentation contains everything you need to know about Scrapy.
Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used
to crawl websites and extract structured data from their pages. It can be used
for a wide range of purposes, from data mining to monitoring and automated
testing.
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
Getting help
============
@ -13,14 +19,14 @@ Having trouble? We'd like to help!
* Try the :doc:`FAQ <faq>` -- it's got answers to some common questions.
* Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`.
* Ask or search questions in `StackOverflow using the scrapy tag`_,
* Search for information in the `archives of the scrapy-users mailing list`_, or
`post a question`_.
* Ask or search questions in `StackOverflow using the scrapy tag`_.
* Ask or search questions in the `Scrapy subreddit`_.
* Search for questions on the archives of the `scrapy-users mailing list`_.
* Ask a question in the `#scrapy IRC channel`_,
* Report bugs with Scrapy in our `issue tracker`_.
.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
.. _issue tracker: https://github.com/scrapy/scrapy/issues
@ -151,21 +157,22 @@ Solving specific problems
topics/contracts
topics/practices
topics/broad-crawls
topics/firefox
topics/firebug
topics/developer-tools
topics/dynamic-content
topics/leaks
topics/media-pipeline
topics/ubuntu
topics/deploy
topics/autothrottle
topics/benchmarking
topics/jobs
topics/coroutines
topics/asyncio
:doc:`faq`
Get answers to most frequently asked questions.
:doc:`topics/debug`
Learn how to debug common problems of your scrapy spider.
Learn how to debug common problems of your Scrapy spider.
:doc:`topics/contracts`
Learn how to use contracts for testing your spiders.
@ -176,11 +183,11 @@ Solving specific problems
:doc:`topics/broad-crawls`
Tune Scrapy for crawling a lot domains in parallel.
:doc:`topics/firefox`
Learn how to scrape with Firefox and some useful add-ons.
:doc:`topics/developer-tools`
Learn how to scrape with your browser's developer tools.
:doc:`topics/firebug`
Learn how to scrape efficiently using Firebug.
:doc:`topics/dynamic-content`
Read webpage data that is loaded dynamically.
:doc:`topics/leaks`
Learn how to find and get rid of memory leaks in your crawler.
@ -188,9 +195,6 @@ Solving specific problems
:doc:`topics/media-pipeline`
Download files and/or images associated with your scraped items.
:doc:`topics/ubuntu`
Install latest Scrapy packages easily on Ubuntu
:doc:`topics/deploy`
Deploying your Scrapy spiders and run them in a remote server.
@ -203,6 +207,12 @@ Solving specific problems
:doc:`topics/jobs`
Learn how to pause and resume crawls for large spiders.
:doc:`topics/coroutines`
Use the :ref:`coroutine syntax <async>`.
:doc:`topics/asyncio`
Use :mod:`asyncio` and :mod:`asyncio`-powered libraries.
.. _extending-scrapy:
Extending Scrapy

209
docs/intro/install.rst
View File

@ -7,18 +7,31 @@ Installation guide
Installing Scrapy
=================
Scrapy runs on Python 2.7 and Python 3.3 or above
(except on Windows where Python 3 is not supported yet).
Scrapy runs on Python 3.5 or above under CPython (default Python
implementation) and PyPy (starting with PyPy 5.9).
If you’re already familiar with installation of Python packages,
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
and macOS.
To install Scrapy using ``conda``, run::
conda install -c conda-forge scrapy
Alternatively, if you’re already familiar with installation of Python packages,
you can install Scrapy and its dependencies from PyPI with::
pip install Scrapy
Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
For more detailed and platform specifics instructions, read on.
For more detailed and platform specifics instructions, as well as
troubleshooting information, read on.
Things that are good to know
@ -52,7 +65,7 @@ please refer to their respective installation instructions:
* `lxml installation`_
* `cryptography installation`_
.. _lxml installation: http://lxml.de/installation.html
.. _lxml installation: https://lxml.de/installation.html
.. _cryptography installation: https://cryptography.io/en/latest/installation/
@ -65,126 +78,81 @@ TL;DR: We recommend installing Scrapy inside a virtual environment
on all platforms.
Python packages can be installed either globally (a.k.a system wide),
or in user-space. We do not recommend installing scrapy system wide.
or in user-space. We do not recommend installing Scrapy system wide.
Instead, we recommend that you install scrapy within a so-called
"virtual environment" (`virtualenv`_).
Virtualenvs allow you to not conflict with already-installed Python
Instead, we recommend that you install Scrapy within a so-called
"virtual environment" (:mod:`venv`).
Virtual environments allow you to not conflict with already-installed Python
system packages (which could break some of your system tools and scripts),
and still install packages normally with ``pip`` (without ``sudo`` and the likes).
To get started with virtual environments, see `virtualenv installation instructions`_.
To install it globally (having it globally installed actually helps here),
it should be a matter of running::
See :ref:`tut-venv` on how to create your virtual environment.
$ [sudo] pip install virtualenv
Check this `user guide`_ on how to create your virtualenv.
.. note::
If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
Once you have created a virtualenv, you can install scrapy inside it with ``pip``,
Once you have created a virtual environment, you can install Scrapy inside it with ``pip``,
just like any other Python package.
(See :ref:`platform-specific guides <intro-install-platform-notes>`
below for non-Python dependencies that you may need to install beforehand).
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
.. _virtualenv: https://virtualenv.pypa.io
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html
.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
.. _intro-install-platform-notes:
Platform specific installation notes
====================================
.. _intro-install-windows:
Windows
-------
* Install Python 2.7 from https://www.python.org/downloads/
Though it's possible to install Scrapy on Windows using pip, we recommend you
to install `Anaconda`_ or `Miniconda`_ and use the package from the
`conda-forge`_ channel, which will avoid most installation issues.
You need to adjust ``PATH`` environment variable to include paths to
the Python executable and additional scripts. The following paths need to be
added to ``PATH``::
Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
C:\Python27\;C:\Python27\Scripts\;
conda install -c conda-forge scrapy
To update the ``PATH`` open a Command prompt and run::
c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py
.. _intro-install-ubuntu:
Close the command prompt window and reopen it so changes take effect, run the
following command and check it shows the expected Python version::
python --version
* Install `pywin32` from http://sourceforge.net/projects/pywin32/
Be sure you download the architecture (win32 or amd64) that matches your system
* *(Only required for Python<2.7.9)* Install `pip`_ from
https://pip.pypa.io/en/latest/installing/
Now open a Command prompt to check ``pip`` is installed correctly::
pip --version
* At this point Python 2.7 and ``pip`` package manager must be working, let's
install Scrapy::
pip install Scrapy
.. note::
Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support
Python 3 on Windows.
Ubuntu 12.04 or above
Ubuntu 14.04 or above
---------------------
Scrapy is currently tested with recent-enough versions of lxml,
twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions.
But it should support older versions of Ubuntu too, like Ubuntu 12.04,
But it should support older versions of Ubuntu too, like Ubuntu 14.04,
albeit with potential issues with TLS connections.
**Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
typically too old and slow to catch up with latest Scrapy.
To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
these dependencies::
sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
are required for ``lxml``
- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
sudo apt-get install python3 python3-dev
Inside a :ref:`virtualenv <intro-using-virtualenv>`,
you can install Scrapy with ``pip`` after that::
pip install scrapy
.. note::
The same non-python dependencies can be used to install Scrapy in Debian
Wheezy (7.0) and above.
The same non-Python dependencies can be used to install Scrapy in Debian
Jessie (8.0) and above.
Mac OS X
--------
.. _intro-install-macos:
macOS
-----
Building Scrapy's dependencies requires the presence of a C compiler and
development headers. On OS X this is typically provided by Apple’s Xcode
development headers. On macOS this is typically provided by Apple’s Xcode
development tools. To install the Xcode command line tools open a terminal
window and run::
@ -199,7 +167,7 @@ solutions:
that doesn't conflict with the rest of your system. Here's how to do it using
the `homebrew`_ package manager:
* Install `homebrew`_ following the instructions in http://brew.sh/
* Install `homebrew`_ following the instructions in https://brew.sh/
* Update your ``PATH`` variable to state that homebrew packages should be
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
@ -220,52 +188,81 @@ solutions:
brew update; brew upgrade python
* *(Optional)* Install Scrapy inside an isolated python environment.
* *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
<intro-using-virtualenv>`.
This method is a workaround for the above OS X issue, but it's an overall
This method is a workaround for the above macOS issue, but it's an overall
good practice for managing dependencies and can complement the first method.
`virtualenv`_ is a tool you can use to create virtual environments in python.
We recommended reading a tutorial like
http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
After any of these workarounds you should be able to install Scrapy::
pip install Scrapy
Anaconda
--------
PyPy
----
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
recommends (only needed when installing Scrapy). Installing on Linux has no special
issues besides installing build dependencies.
Installing Scrapy with PyPy on Windows is not tested.
You can check that Scrapy is installed correctly by running ``scrapy bench``.
If this command gives errors such as
``TypeError: ... got 2 unexpected keyword arguments``, this means
that setuptools was unable to pick up one PyPy-specific dependency.
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
Using Anaconda is an alternative to using a virtualenv and installing with ``pip``.
.. _intro-install-troubleshooting:
.. note::
Troubleshooting
===============
For Windows users, or if you have issues installing through ``pip``, this is
the recommended way to install Scrapy.
AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
----------------------------------------------------------------
If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_
community have up-to-date packages for Linux, Windows and OS X.
After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an
exception with the following traceback::
To install Scrapy using ``conda``, run::
[…]
File "[…]/site-packages/twisted/protocols/tls.py", line 63, in <module>
from twisted.internet._sslverify import _setAcceptableProtocols
File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in <module>
TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
conda install -c conda-forge scrapy
The reason you get this exception is that your system or virtual environment
has a version of pyOpenSSL that your version of Twisted does not support.
To install a version of pyOpenSSL that your version of Twisted supports,
reinstall Twisted with the :code:`tls` extra option::
pip install twisted[tls]
For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
.. _Python: https://www.python.org/
.. _pip: https://pip.pypa.io/en/latest/installing/
.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
.. _lxml: http://lxml.de/
.. _parsel: https://pypi.python.org/pypi/parsel
.. _w3lib: https://pypi.python.org/pypi/w3lib
.. _twisted: https://twistedmatrix.com/
.. _cryptography: https://cryptography.io/
.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
.. _lxml: https://lxml.de/index.html
.. _parsel: https://pypi.org/project/parsel/
.. _w3lib: https://pypi.org/project/w3lib/
.. _twisted: https://twistedmatrix.com/trac/
.. _cryptography: https://cryptography.io/en/latest/
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
.. _setuptools: https://pypi.python.org/pypi/setuptools
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
.. _homebrew: http://brew.sh/
.. _zsh: http://www.zsh.org/
.. _Scrapinghub: http://scrapinghub.com
.. _Anaconda: http://docs.continuum.io/anaconda/index
.. _Miniconda: http://conda.pydata.org/docs/install/quick.html
.. _conda-forge: https://conda-forge.github.io/
.. _homebrew: https://brew.sh/
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _conda-forge: https://conda-forge.org/

15
docs/intro/overview.rst
View File

@ -26,7 +26,7 @@ http://quotes.toscrape.com, following the pagination::
class QuotesSpider(scrapy.Spider):
name = "quotes"
name = 'quotes'
start_urls = [
'http://quotes.toscrape.com/tag/humor/',
]
@ -34,14 +34,13 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').extract_first(),
'author': quote.xpath('span/small/text()').extract_first(),
'author': quote.xpath('span/small/text()').get(),
'text': quote.css('span.text::text').get(),
}
next_page = response.css('li.next a::attr("href")').extract_first()
next_page = response.css('li.next a::attr("href")').get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
@ -161,8 +160,8 @@ The next steps for you are to :ref:`install Scrapy <intro-install>`,
a full-blown Scrapy project and `join the community`_. Thanks for your
interest!
.. _join the community: http://scrapy.org/community/
.. _join the community: https://scrapy.org/community/
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
.. _Amazon S3: https://aws.amazon.com/s3/
.. _Sitemaps: http://www.sitemaps.org
.. _Sitemaps: https://www.sitemaps.org/index.html

332
docs/intro/tutorial.rst
View File

@ -22,19 +22,27 @@ Scrapy is written in Python_. If you're new to the language you might want to
start by getting an idea of what the language is like, to get the most out of
Scrapy.
If you're already familiar with other languages, and want to learn Python
quickly, we recommend reading through `Dive Into Python 3`_. Alternatively,
you can follow the `Python Tutorial`_.
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
If you're new to programming and want to start with Python, you may find useful
the online book `Learn Python The Hard Way`_. You can also take a look at `this
list of Python resources for non-programmers`_.
If you're new to programming and want to start with Python, the following books
may be useful to you:
* `Automate the Boring Stuff With Python`_
* `How To Think Like a Computer Scientist`_
* `Learn Python 3 The Hard Way`_
You can also take a look at `this list of Python resources for non-programmers`_,
as well as the `suggested resources in the learnpython-subreddit`_.
.. _Python: https://www.python.org/
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
.. _Dive Into Python 3: http://www.diveintopython3.net
.. _Python Tutorial: https://docs.python.org/3/tutorial
.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
.. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
.. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/
.. _Learn Python 3 The Hard Way: https://learnpythonthehardway.org/python3/
.. _suggested resources in the learnpython-subreddit: https://www.reddit.com/r/learnpython/wiki/index#wiki_new_to_python.3F
Creating a project
@ -54,6 +62,8 @@ This will create a ``tutorial`` directory with the following contents::
__init__.py
items.py # project items definition file
middlewares.py # project middlewares file
pipelines.py # project pipelines file
@ -68,9 +78,9 @@ Our first Spider
Spiders are classes that you define and that Scrapy uses to scrape information
from a website (or a group of websites). They must subclass
:class:`scrapy.Spider` and define the initial requests to make, optionally how
to follow links in the pages, and how to parse the downloaded page content to
extract data.
:class:`~scrapy.spiders.Spider` and define the initial requests to make,
optionally how to follow links in the pages, and how to parse the downloaded
page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
@ -130,15 +140,15 @@ will send some requests for the ``quotes.toscrape.com`` domain. You will get an
similar to this::
... (omitted for brevity)
2016-09-20 14:48:00 [scrapy] INFO: Spider opened
2016-09-20 14:48:00 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2016-09-20 14:48:00 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
2016-09-20 14:48:00 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-1.html
2016-09-20 14:48:01 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
2016-09-20 14:48:01 [quotes] DEBUG: Saved file quotes-2.html
2016-09-20 14:48:01 [scrapy] INFO: Closing spider (finished)
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
...
Now, check the files in the current directory. You should notice that two new
@ -195,14 +205,14 @@ Extracting data
---------------
The best way to learn how to extract data with Scrapy is trying selectors
using the shell :ref:`Scrapy shell <topics-shell>`. Run::
using the :ref:`Scrapy shell <topics-shell>`. Run::
scrapy shell 'http://quotes.toscrape.com/page/1/'
.. note::
Remember to always enclose urls in quotes when running Scrapy shell from
command-line, otherwise urls containing arguments (ie. ``&`` character)
command-line, otherwise urls containing arguments (i.e. ``&`` character)
will not work.
On Windows, use double quotes instead::
@ -212,7 +222,7 @@ using the shell :ref:`Scrapy shell <topics-shell>`. Run::
You will see something like::
[ ... Scrapy log here ... ]
2016-09-19 12:09:27 [scrapy] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
[s] Available Scrapy objects:
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
@ -225,13 +235,16 @@ You will see something like::
[s] shelp() Shell help (print this help)
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
[s] view(response) View response in a browser
>>>
Using the shell, you can try selecting elements using `CSS`_ with the response
object::
object:
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@ -239,71 +252,72 @@ The result of running ``response.css('title')`` is a list-like object called
and allow you to run further queries to fine-grain the selection or extract the
data.
To extract the text from the title above, you can do::
To extract the text from the title above, you can do:
>>> response.css('title::text').extract()
['Quotes to Scrape']
>>> response.css('title::text').getall()
['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
CSS query, to mean we want to select only the text elements directly inside
``<title>`` element. If we don't specify ``::text``, we'd get the full title
element, including its tags::
element, including its tags:
>>> response.css('title').extract()
['<title>Quotes to Scrape</title>']
>>> response.css('title').getall()
['<title>Quotes to Scrape</title>']
The other thing is that the result of calling ``.extract()`` is a list, because
we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When
you know you just want the first result, as in this case, you can do::
The other thing is that the result of calling ``.getall()`` is a list: it is
possible that a selector returns more than one result, so we extract them all.
When you know you just want the first result, as in this case, you can do:
>>> response.css('title::text').extract_first()
'Quotes to Scrape'
>>> response.css('title::text').get()
'Quotes to Scrape'
As an alternative, you could've written::
As an alternative, you could've written:
>>> response.css('title::text')[0].extract()
'Quotes to Scrape'
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
However, using ``.extract_first()`` avoids an ``IndexError`` and returns
``None`` when it doesn't find any element matching the selection.
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
find any element matching the selection.
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
to be scraped, you can at least get **some** data.
Besides the :meth:`~scrapy.selector.Selector.extract` and
:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
expressions`::
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
expressions`_:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
In order to find the proper CSS selectors to use, you might find useful opening
the response page from the shell in your web browser using ``view(response)``.
You can use your browser developer tools or extensions like Firebug (see
sections about :ref:`topics-firebug` and :ref:`topics-firefox`).
You can use your browser's developer tools to inspect the HTML and come up
with a selector (see :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
.. _regular expressions: https://docs.python.org/3/library/re.html
.. _Selector Gadget: http://selectorgadget.com/
.. _Selector Gadget: https://selectorgadget.com/
XPath: a brief intro
^^^^^^^^^^^^^^^^^^^^
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').extract_first()
'Quotes to Scrape'
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').get()
'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@ -323,7 +337,7 @@ recommend `this tutorial to learn XPath through examples
<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
.. _XPath: https://www.w3.org/TR/xpath
.. _XPath: https://www.w3.org/TR/xpath/all/
.. _CSS: https://www.w3.org/TR/selectors
Extracting quotes and authors
@ -358,48 +372,56 @@ we want::
$ scrapy shell 'http://quotes.toscrape.com'
We get a list of selectors for the quote HTML elements with::
We get a list of selectors for the quote HTML elements with:
>>> response.css("div.quote")
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
variable, so that we can run our CSS selectors directly on a particular quote::
variable, so that we can run our CSS selectors directly on a particular quote:
>>> quote = response.css("div.quote")[0]
>>> quote = response.css("div.quote")[0]
Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created::
Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created:
>>> title = quote.css("span.text::text").extract_first()
>>> title
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").extract_first()
>>> author
'Albert Einstein'
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
Given that the tags are a list of strings, we can use the ``.extract()`` method
to get all of them::
Given that the tags are a list of strings, we can use the ``.getall()`` method
to get all of them:
>>> tags = quote.css("div.tags a.tag::text").extract()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. invisible-code-block: python
from sys import version_info
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary::
quotes elements and put them together into a Python dictionary:
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").extract_first()
... author = quote.css("small.author::text").extract_first()
... tags = quote.css("div.tags a.tag::text").extract()
... print(dict(text=text, author=author, tags=tags))
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
... a few more of these, omitted for brevity
>>>
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
Extracting data in our spider
------------------------------
-----------------------------
Let's get back to our spider. Until now, it doesn't extract any data in
particular, just saves the whole HTML page to a local file. Let's integrate the
@ -422,16 +444,16 @@ in the callback, as you can see below::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').extract_first(),
'author': quote.css('span small::text').extract_first(),
'tags': quote.css('div.tags a.tag::text').extract(),
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
If you run this spider, it will output the extracted data with the log::
2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
2016-09-19 18:57:19 [scrapy] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
@ -452,7 +474,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting
its contents. If you run this command twice without removing the file
before the second time, you'll end up with a broken JSON file.
You can also used other formats, like `JSON Lines`_::
You can also use other formats, like `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
@ -494,17 +516,23 @@ markup:
</li>
</ul>
We can try extracting it in the shell::
We can try extracting it in the shell:
>>> response.css('li.next a').extract_first()
'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
>>> response.css('li.next a').get()
'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
This gets the anchor element, but we want the attribute ``href``. For that,
Scrapy supports a CSS extension that let's you select the attribute contents,
like this::
Scrapy supports a CSS extension that lets you select the attribute contents,
like this:
>>> response.css('li.next a::attr(href)').extract_first()
'/page/2/'
>>> response.css('li.next a::attr(href)').get()
'/page/2/'
There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more):
>>> response.css('li.next a').attrib['href']
'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
@ -521,12 +549,12 @@ page, extracting data from it::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').extract_first(),
'author': quote.css('span small::text').extract_first(),
'tags': quote.css('div.tags a.tag::text').extract(),
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
next_page = response.css('li.next a::attr(href)').extract_first()
next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
@ -551,13 +579,69 @@ In our example, it creates a sort of loop, following all the links to the next p
until it doesn't find one -- handy for crawling blogs, forums and other sites with
pagination.
.. _response-follow-example:
A shortcut for creating Requests
--------------------------------
As a shortcut for creating Request objects you can use
:meth:`response.follow <scrapy.http.TextResponse.follow>`::
import scrapy
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
]
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('span small::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no
need to call urljoin. Note that ``response.follow`` just returns a Request
instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
for href in response.css('ul.pager a::attr(href)'):
yield response.follow(href, callback=self.parse)
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
for a in response.css('ul.pager a'):
yield response.follow(a, callback=self.parse)
To create multiple requests from an iterable, you can use
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
anchors = response.css('ul.pager a')
yield from response.follow_all(anchors, callback=self.parse)
or, shortening it further::
yield from response.follow_all(css='ul.pager a', callback=self.parse)
More examples and patterns
--------------------------
Here is another spider that illustrates callbacks and following links,
this time for scraping author information::
import scrapy
@ -567,20 +651,15 @@ this time for scraping author information::
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
# follow links to author pages
for href in response.css('.author+a::attr(href)').extract():
yield scrapy.Request(response.urljoin(href),
callback=self.parse_author)
author_page_links = response.css('.author + a')
yield from response.follow_all(author_page_links, self.parse_author)
# follow pagination links
next_page = response.css('li.next a::attr(href)').extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
pagination_links = response.css('li.next a')
yield from response.follow_all(pagination_links, self.parse)
def parse_author(self, response):
def extract_with_css(query):
return response.css(query).extract_first().strip()
return response.css(query).get(default='').strip()
yield {
'name': extract_with_css('h3.author-title::text'),
@ -592,6 +671,11 @@ This spider will start from the main page, it will follow all the links to the
authors pages calling the ``parse_author`` callback for each of them, and also
the pagination links with the ``parse`` callback as we saw before.
Here we're passing callbacks to
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
arguments to make the code shorter; it also works for
:class:`~scrapy.http.Request`.
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.
@ -624,7 +708,7 @@ option when running them::
scrapy crawl quotes -o quotes-humor.json -a tag=humor
These arguments are passed to the Spider's ``__init__`` method and become
spider attributes by default.
spider attributes by default.
In this example, the value provided for the ``tag`` argument will be available
via ``self.tag``. You can use this to make your spider fetch only quotes
@ -646,14 +730,13 @@ with a specific tag, building the URL based on the argument::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').extract_first(),
'author': quote.css('span small a::text').extract_first(),
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
}
next_page = response.css('li.next a::attr(href)').extract_first()
next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, self.parse)
yield response.follow(next_page, self.parse)
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
@ -675,4 +758,3 @@ modeling the scraped data. If you prefer to play with an example project, check
the :ref:`intro-examples` section.
.. _JSON: https://en.wikipedia.org/wiki/JSON
.. _dirbot: https://github.com/scrapy/dirbot

2073
docs/news.rst
View File

File diff suppressed because it is too large Load Diff

4
docs/requirements.txt Normal file
View File

@ -0,0 +1,4 @@
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page
sphinx_rtd_theme

BIN
docs/topics/_images/firebug1.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 43 KiB

BIN
docs/topics/_images/firebug2.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 68 KiB

BIN
docs/topics/_images/firebug3.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 88 KiB

BIN
docs/topics/_images/inspector_01.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 53 KiB

BIN
docs/topics/_images/network_01.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

BIN
docs/topics/_images/network_02.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 81 KiB

BIN
docs/topics/_images/network_03.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 44 KiB

13
docs/topics/api.rst
View File

@ -94,11 +94,13 @@ how you :ref:`configure the downloader middlewares
.. method:: crawl(\*args, \**kwargs)
Starts the crawler by instantiating its spider class with the given
`args` and `kwargs` arguments, while setting the execution engine in
``args`` and ``kwargs`` arguments, while setting the execution engine in
motion.
Returns a deferred that is fired when the crawl is finished.
.. automethod:: stop
.. autoclass:: CrawlerRunner
:members:
@ -154,7 +156,7 @@ Settings API
SpiderLoader API
================
.. module:: scrapy.loader
.. module:: scrapy.spiderloader
:synopsis: The spider loader
.. class:: SpiderLoader
@ -171,7 +173,8 @@ SpiderLoader API
This class method is used by Scrapy to create an instance of the class.
It's called with the current project settings, and it loads the spiders
found in the modules of the :setting:`SPIDER_MODULES` setting.
found recursively in the modules of the :setting:`SPIDER_MODULES`
setting.
:param settings: project settings
:type settings: :class:`~scrapy.settings.Settings` instance
@ -179,7 +182,7 @@ SpiderLoader API
.. method:: load(spider_name)
Get the Spider class with the given name. It'll look into the previously
loaded spiders for a spider class with name `spider_name` and will raise
loaded spiders for a spider class with name ``spider_name`` and will raise
a KeyError if not found.
:param spider_name: spider class name
@ -270,5 +273,3 @@ class (which they all inherit from).
Close the given spider. After this is called, no more specific stats
can be accessed or collected.
.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html

7
docs/topics/architecture.rst
View File

@ -12,7 +12,7 @@ Overview
The following diagram shows an overview of the Scrapy architecture with its
components and an outline of the data flow that takes place inside the system
(shown by the green arrows). A brief description of the components is included
(shown by the red arrows). A brief description of the components is included
below with links for more detailed information about them. The data flow is
also described below.
@ -166,11 +166,10 @@ for concurrency.
For more information about asynchronous programming and Twisted see these
links:
* `Introduction to Deferreds in Twisted`_
* :doc:`twisted:core/howto/defer-intro`
* `Twisted - hello, asynchronous programming`_
* `Twisted Introduction - Krondo`_
.. _Twisted: https://twistedmatrix.com/trac/
.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/

28
docs/topics/asyncio.rst Normal file
View File

@ -0,0 +1,28 @@
=======
asyncio
=======
.. versionadded:: 2.0
Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related changes without a deprecation
period or warning.
.. _install-asyncio:
Installing the asyncio reactor
==============================
To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to
``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.
If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to
install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`::
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')

3
docs/topics/autothrottle.rst
View File

@ -11,7 +11,7 @@ Design goals
============
1. be nicer to sites instead of using default download delay of zero
2. automatically adjust scrapy to the optimum crawling speed, so the user
2. automatically adjust Scrapy to the optimum crawling speed, so the user
doesn't have to tune the download delays to find the optimum one.
The user only needs to specify the maximum concurrent requests
it allows, and the extension does the rest.
@ -88,6 +88,7 @@ The settings used to control the AutoThrottle extension are:
* :setting:`AUTOTHROTTLE_ENABLED`
* :setting:`AUTOTHROTTLE_START_DELAY`
* :setting:`AUTOTHROTTLE_MAX_DELAY`
* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
* :setting:`AUTOTHROTTLE_DEBUG`
* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
* :setting:`CONCURRENT_REQUESTS_PER_IP`

92
docs/topics/benchmarking.rst
View File

@ -18,40 +18,66 @@ To run it use::
You should see an output like this::
2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot)
2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened
2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout)
2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 231508,
'downloader/request_count': 682,
'downloader/request_method_count/GET': 682,
'downloader/response_bytes': 1172802,
'downloader/response_count': 682,
'downloader/response_status_count/200': 682,
'finish_reason': 'closespider_timeout',
'finish_time': datetime.datetime(2013, 5, 16, 16, 8, 57, 985539),
'log_count/INFO': 14,
'request_depth_max': 34,
'response_received_count': 682,
'scheduler/dequeued': 682,
'scheduler/dequeued/memory': 682,
'scheduler/enqueued': 12767,
'scheduler/enqueued/memory': 12767,
'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)}
2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout)
2016-12-16 21:18:48 [scrapy.utils.log] INFO: Scrapy 1.2.2 started (bot: quotesbot)
2016-12-16 21:18:48 [scrapy.utils.log] INFO: Overridden settings: {'CLOSESPIDER_TIMEOUT': 10, 'ROBOTSTXT_OBEY': True, 'SPIDER_MODULES': ['quotesbot.spiders'], 'LOGSTATS_INTERVAL': 1, 'BOT_NAME': 'quotesbot', 'LOG_LEVEL': 'INFO', 'NEWSPIDER_MODULE': 'quotesbot.spiders'}
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled extensions:
['scrapy.extensions.closespider.CloseSpider',
'scrapy.extensions.logstats.LogStats',
'scrapy.extensions.telnet.TelnetConsole',
'scrapy.extensions.corestats.CoreStats']
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares:
['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
'scrapy.downloadermiddlewares.retry.RetryMiddleware',
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
'scrapy.downloadermiddlewares.stats.DownloaderStats']
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled item pipelines:
[]
2016-12-16 21:18:49 [scrapy.core.engine] INFO: Spider opened
2016-12-16 21:18:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:50 [scrapy.extensions.logstats] INFO: Crawled 70 pages (at 4200 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:51 [scrapy.extensions.logstats] INFO: Crawled 134 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:52 [scrapy.extensions.logstats] INFO: Crawled 198 pages (at 3840 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:53 [scrapy.extensions.logstats] INFO: Crawled 254 pages (at 3360 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:54 [scrapy.extensions.logstats] INFO: Crawled 302 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:55 [scrapy.extensions.logstats] INFO: Crawled 358 pages (at 3360 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:56 [scrapy.extensions.logstats] INFO: Crawled 406 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:57 [scrapy.extensions.logstats] INFO: Crawled 438 pages (at 1920 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:58 [scrapy.extensions.logstats] INFO: Crawled 470 pages (at 1920 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:18:59 [scrapy.core.engine] INFO: Closing spider (closespider_timeout)
2016-12-16 21:18:59 [scrapy.extensions.logstats] INFO: Crawled 518 pages (at 2880 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:19:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 229995,
'downloader/request_count': 534,
'downloader/request_method_count/GET': 534,
'downloader/response_bytes': 1565504,
'downloader/response_count': 534,
'downloader/response_status_count/200': 534,
'finish_reason': 'closespider_timeout',
'finish_time': datetime.datetime(2016, 12, 16, 16, 19, 0, 647725),
'log_count/INFO': 17,
'request_depth_max': 19,
'response_received_count': 534,
'scheduler/dequeued': 533,
'scheduler/dequeued/memory': 533,
'scheduler/enqueued': 10661,
'scheduler/enqueued/memory': 10661,
'start_time': datetime.datetime(2016, 12, 16, 16, 18, 49, 799869)}
2016-12-16 21:19:00 [scrapy.core.engine] INFO: Spider closed (closespider_timeout)
That tells you that Scrapy is able to crawl about 3900 pages per minute in the
That tells you that Scrapy is able to crawl about 3000 pages per minute in the
hardware where you run it. Note that this is a very simple spider intended to
follow links, any custom spider you write will probably do more stuff which
results in slower crawl rates. How slower depends on how much your spider does

77
docs/topics/broad-crawls.rst
View File

@ -20,7 +20,7 @@ These are some common properties often found in broad crawls:
* they crawl many domains (often, unbounded) instead of a specific set of sites
* they don't necessarily crawl domains to completion, because it would
* they don't necessarily crawl domains to completion, because it would be
impractical (or impossible) to do so, and instead limit the crawl by time or
number of pages crawled
@ -39,24 +39,50 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
concrete suggestions of Scrapy settings to tune in order to achieve an
efficient broad crawl.
.. _broad-crawls-scheduler-priority-queue:
Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
=================================================
Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
It works best during single-domain crawl. It does not work well with crawling
many different domains in parallel
To apply the recommended priority queue use::
SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
.. _broad-crawls-concurrency:
Increase concurrency
====================
Concurrency is the number of requests that are processed in parallel. There is
a global limit and a per-domain limit.
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
.. note:: The scheduler priority queue :ref:`recommended for broad crawls
<broad-crawls-scheduler-priority-queue>` does not support
:setting:`CONCURRENT_REQUESTS_PER_IP`.
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
to increase it will depend on how much CPU you crawler will have available. A
good starting point is ``100``, but the best way to find out is by doing some
trials and identifying at what concurrency your Scrapy process gets CPU
bounded. For optimum performance, you should pick a concurrency where CPU usage
is at 80-90%.
to increase it will depend on how much CPU and memory you crawler will have
available.
To increase the global concurrency use::
A good starting point is ``100``::
CONCURRENT_REQUESTS = 100
But the best way to find out is by doing some trials and identifying at what
concurrency your Scrapy process gets CPU bounded. For optimum performance, you
should pick a concurrency where CPU usage is at 80-90%.
Increasing concurrency also increases memory usage. If memory usage is a
concern, you might need to lower your global concurrency limit accordingly.
Increase Twisted IO thread pool maximum size
============================================
@ -85,8 +111,8 @@ When doing broad crawls you are often only interested in the crawl rates you
get and any errors found. These stats are reported by Scrapy when using the
``INFO`` log level. In order to save CPU (and log storage requirements) you
should not use ``DEBUG`` log level when preforming large broad crawls in
production. Using ``DEBUG`` level when developing your (broad) crawler may fine
though.
production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::
@ -162,4 +188,33 @@ AjaxCrawlMiddleware helps to crawl them correctly.
It is turned OFF by default because it has some performance overhead,
and enabling it for focused crawls doesn't make much sense.
.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
.. _broad-crawls-bfo:
Crawl in BFO order
==================
:ref:`Scrapy crawls in DFO order by default <faq-bfo-dfo>`.
In broad crawls, however, page crawling tends to be faster than page
processing. As a result, unprocessed early requests stay in memory until the
final depth is reached, which can significantly increase memory usage.
:ref:`Crawl in BFO order <faq-bfo-dfo>` instead to save memory.
Be mindful of memory leaks
==========================
If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
<broad-crawls-concurrency>` you should :ref:`debug your memory leaks
<topics-leaks>`.
Install a specific Twisted reactor
==================================
If the crawl is exceeding the system's capabilities, you might want to try
installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.

117
docs/topics/commands.rst
View File

@ -1,3 +1,5 @@
.. highlight:: none
.. _topics-commands:
=================
@ -27,7 +29,7 @@ in standard locations:
1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
for global (user-wide) settings, and
3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
Settings from these files are merged in the listed order of preference:
user-defined values have higher priority than system-wide defaults
@ -37,7 +39,7 @@ Scrapy also understands, and can be configured through, a number of environment
variables. Currently these are:
* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
* ``SCRAPY_PROJECT``
* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`)
* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
.. _topics-project-structure:
@ -55,6 +57,7 @@ structure by default, similar to this::
myproject/
__init__.py
items.py
middlewares.py
pipelines.py
settings.py
spiders/
@ -65,11 +68,42 @@ structure by default, similar to this::
The directory where the ``scrapy.cfg`` file resides is known as the *project
root directory*. That file contains the name of the python module that defines
the project settings. Here is an example::
the project settings. Here is an example:
.. code-block:: ini
[settings]
default = myproject.settings
.. _topics-project-envvar:
Sharing the root directory between projects
===========================================
A project root directory, the one that contains the ``scrapy.cfg``, may be
shared by multiple Scrapy projects, each with its own settings module.
In that case, you must define one or more aliases for those settings modules
under ``[settings]`` in your ``scrapy.cfg`` file:
.. code-block:: ini
[settings]
default = myproject1.settings
project1 = myproject1.settings
project2 = myproject2.settings
By default, the ``scrapy`` command-line tool will use the ``default`` settings.
Use the ``SCRAPY_PROJECT`` environment variable to specify a different project
for ``scrapy`` to use::
$ scrapy settings --get BOT_NAME
Project 1 Bot
$ export SCRAPY_PROJECT=project2
$ scrapy settings --get BOT_NAME
Project 2 Bot
Using the ``scrapy`` tool
=========================
@ -187,7 +221,7 @@ startproject
Creates a new Scrapy project named ``project_name``, under the ``project_dir``
directory.
If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``.
If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``.
Usage example::
@ -249,6 +283,8 @@ check
Run contract checks.
.. skip: start
Usage examples::
$ scrapy check -l
@ -266,6 +302,8 @@ Usage examples::
[FAILED] first_spider:parse
>>> Returned 92 requests, expected 0..4
.. skip: end
.. command:: list
list
@ -291,12 +329,12 @@ edit
* Syntax: ``scrapy edit <spider>``
* Requires project: *yes*
Edit the given spider using the editor defined in the :setting:`EDITOR`
setting.
Edit the given spider using the editor defined in the ``EDITOR`` environment
variable or (if unset) the :setting:`EDITOR` setting.
This command is provided only as a convenience shortcut for the most common
case, the developer is of course free to choose any tool or IDE to write and
debug his spiders.
debug spiders.
Usage example::
@ -322,6 +360,14 @@ So this command can be used to "see" how your spider would fetch a certain page.
If used outside a project, no particular per-spider behaviour would be applied
and it will just use the default Scrapy downloader settings.
Supported options:
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
* ``--headers``: print the response's HTTP headers instead of the response's body
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
Usage examples::
$ scrapy fetch --nolog http://www.example.com/some/page.html
@ -350,6 +396,12 @@ Opens the given URL in a browser, as your Scrapy spider would "see" it.
Sometimes spiders see pages differently from regular users, so this can be used
to check what the spider "sees" and confirm it's what you expect.
Supported options:
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
Usage example::
$ scrapy view http://www.example.com/some/page.html
@ -368,11 +420,34 @@ given. Also supports UNIX-style local file paths, either relative with
``./`` or ``../`` prefixes or absolute file paths.
See :ref:`topics-shell` for more info.
Supported options:
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
* ``-c code``: evaluate the code in the shell, print the result and exit
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them);
this only affects the URL you may pass as argument on the command line;
once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default.
Usage example::
$ scrapy shell http://www.example.com/some/page.html
[ ... scrapy shell starts ... ]
$ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)'
(200, 'http://www.example.com/')
# shell follows HTTP redirects by default
$ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
(200, 'http://example.com/')
# you can disable this with --no-redirect
# (only for the URL passed as command line argument)
$ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)'
(302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F')
.. command:: parse
parse
@ -393,6 +468,12 @@ Supported options:
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
response
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
* ``--pipelines``: process items through pipelines
* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
@ -410,6 +491,8 @@ Supported options:
* ``--verbose`` or ``-v``: display information for each depth level
.. skip: start
Usage example::
$ scrapy parse http://www.example.com/ -c parse_item
@ -417,13 +500,15 @@ Usage example::
>>> STATUS DEPTH LEVEL 1 <<<
# Scraped Items ------------------------------------------------------------
[{'name': u'Example item',
'category': u'Furniture',
'length': u'12 cm'}]
[{'name': 'Example item',
'category': 'Furniture',
'length': '12 cm'}]
# Requests -----------------------------------------------------------------
[]
.. skip: end
.. command:: settings
@ -502,11 +587,13 @@ Default: ``''`` (empty string)
A module to use for looking up custom Scrapy commands. This is used to add custom
commands for your Scrapy project.
Example::
Example:
.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
.. _Deploying your project: http://scrapyd.readthedocs.org/en/latest/deploy.html
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
Register commands via setup.py entry points
-------------------------------------------
@ -517,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.
The following example adds ``my_command`` command::
The following example adds ``my_command`` command:
.. skip: next
.. code-block:: python
from setuptools import setup, find_packages

55
docs/topics/contracts.rst
View File

@ -6,10 +6,6 @@ Spiders Contracts
.. versionadded:: 0.15
.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
to minor functionality/API updates. Check the :ref:`release notes <news>` to
be notified of updates.
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@ -35,12 +31,20 @@ This callback is tested using three built-in contracts:
.. class:: UrlContract
This contract (``@url``) sets the sample url used when checking other
This contract (``@url``) sets the sample URL used when checking other
contract conditions for this spider. This contract is mandatory. All
callbacks lacking this contract are ignored when running the checks::
@url url
.. class:: CallbackKeywordArgumentsContract
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
attribute for the sample request. It must be a valid JSON dictionary.
::
@cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
.. class:: ReturnsContract
This contract (``@returns``) sets lower and upper bounds for the items and
@ -60,7 +64,7 @@ Use the :command:`check` command to run the contract checks.
Custom Contracts
================
If you find you need more power than the built-in scrapy contracts you can
If you find you need more power than the built-in Scrapy contracts you can
create and load your own contracts in the project by using the
:setting:`SPIDER_CONTRACTS` setting::
@ -69,7 +73,7 @@ create and load your own contracts in the project by using the
'myproject.contracts.ItemValidate': 10,
}
Each contract must inherit from :class:`scrapy.contracts.Contract` and can
Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
override three methods:
.. module:: scrapy.contracts
@ -86,8 +90,11 @@ override three methods:
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
for :class:`~scrapy.http.Request` object. Must return the same or a
modified version of it.
for request object. :class:`~scrapy.http.Request` is used by default,
but this can be changed with the ``request_cls`` attribute.
If multiple contracts in chain have this attribute defined, the last one is used.
Must return the same or a modified version of it.
.. method:: Contract.pre_process(response)
@ -99,9 +106,14 @@ override three methods:
This allows processing the output of the callback. Iterators are
converted listified before being passed to this hook.
Raise :class:`~scrapy.exceptions.ContractFail` from
:class:`~scrapy.contracts.Contract.pre_process` or
:class:`~scrapy.contracts.Contract.post_process` if expectations are not met:
.. autoclass:: scrapy.exceptions.ContractFail
Here is a demo contract which checks the presence of a custom header in the
response received. Raise :class:`scrapy.exceptions.ContractFail` in order to
get the failures pretty printed::
response received::
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
@ -117,3 +129,24 @@ get the failures pretty printed::
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
.. _detecting-contract-check-runs:
Detecting check runs
====================
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use `os.environ`_ to perform any change to
your spiders or your settings when ``scrapy check`` is used::
import os
import scrapy
class ExampleSpider(scrapy.Spider):
name = 'example'
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
pass # Do some scraper adjustments when a check is running
.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

110
docs/topics/coroutines.rst Normal file
View File

@ -0,0 +1,110 @@
==========
Coroutines
==========
.. versionadded:: 2.0
Scrapy has :ref:`partial support <coroutine-support>` for the
:ref:`coroutine syntax <async>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related API and behavior changes without a
deprecation period or warning.
.. _coroutine-support:
Supported callables
===================
The following callables may be defined as coroutines using ``async def``, and
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.http.Request` callbacks.
The following are known caveats of the current implementation that we aim
to address in future versions of Scrapy:
- The callback output is not processed until the whole callback finishes.
As a side effect, if the callback raises an exception, none of its
output is processed.
- Because `asynchronous generators were introduced in Python 3.6`_, you
can only use ``yield`` if you are using Python 3.6 or later.
If you need to output multiple items or requests and you are using
Python 3.5, return an iterable (e.g. a list) instead.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
- The
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`,
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`,
and
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
methods of
:ref:`downloader middlewares <topics-downloader-middleware-custom>`.
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
Usage
=====
There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
class DbPipeline:
def _update_item(self, data, item):
item['field'] = data
return item
def process_item(self, item, spider):
dfd = db.get_some_data(item['id'])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
class DbPipeline:
async def process_item(self, item, spider):
item['field'] = await db.get_some_data(item['id'])
return item
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
many useful Python libraries providing such code::
class MySpider(Spider):
# ...
async def parse_with_deferred(self, response):
additional_response = await treq.get('https://additional.url')
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
async def parse_with_asyncio(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
additional_data = await r.text()
# ... use response and additional_data to yield items and requests
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
:mod:`asyncio` loop and to use them you need to
:doc:`enable asyncio support in Scrapy<asyncio>`.
Common use cases for asynchronous code include:
* requesting data from websites, databases and other services (in callbacks,
pipelines and middlewares);
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs
.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable

29
docs/topics/debug.rst
View File

@ -5,7 +5,7 @@ Debugging Spiders
=================
This document explains the most common techniques for debugging spiders.
Consider the following scrapy spider below::
Consider the following Scrapy spider below::
import scrapy
from myproject.items import MyItem
@ -18,24 +18,25 @@ Consider the following scrapy spider below::
)
def parse(self, response):
# collect `item_urls`
# <processing code not shown>
# collect `item_urls`
for item_url in item_urls:
yield scrapy.Request(item_url, self.parse_item)
def parse_item(self, response):
# <processing code not shown>
item = MyItem()
# populate `item` fields
# and extract item_details_url
yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
def parse_details(self, response):
item = response.meta['item']
def parse_details(self, response, item):
# populate more `item` fields
return item
Basically this is a simple spider which parses two pages of items (the
start_urls). Items also have a details page with additional information, so we
use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
partially populated item.
@ -47,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
of the spider at the method level. It has the advantage of being flexible and
simple to use, but does not allow debugging code inside a method.
.. highlight:: none
.. skip: start
In order to see the item scraped from a specific url::
$ scrapy parse --spider=myspider -c parse_item -d 2 <item_url>
@ -84,6 +89,8 @@ using::
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
.. skip: end
Scrapy Shell
============
@ -93,13 +100,14 @@ spider, it is of little help to check what happens inside a callback, besides
showing the response received and the output. How to debug the situation when
``parse_details`` sometimes receives no item?
.. highlight:: python
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
from scrapy.shell import inspect_response
def parse_details(self, response):
item = response.meta.get('item', None)
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
@ -132,8 +140,7 @@ Logging is another useful option for getting information about your spider run.
Although not as convenient, it comes with the advantage that the logs will be
available in all future runs should they be necessary again::
def parse_details(self, response):
item = response.meta.get('item', None)
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
@ -142,4 +149,4 @@ available in all future runs should they be necessary again::
For more information, check the :ref:`topics-logging` section.
.. _base tag: http://www.w3schools.com/tags/tag_base.asp
.. _base tag: https://www.w3schools.com/tags/tag_base.asp

12
docs/topics/deploy.rst
View File

@ -50,10 +50,10 @@ them as needed - the configuration is read from the ``scrapy.cfg`` file
just like ``scrapyd-deploy``.
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html
.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
.. _shub: http://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: http://scrapyd.readthedocs.org/en/latest/deploy.html
.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: http://scrapinghub.com/
.. _shub: https://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: https://scrapinghub.com/

306
docs/topics/developer-tools.rst Normal file
View File

@ -0,0 +1,306 @@
.. _topics-developer-tools:
=================================================
Using your browser's Developer Tools for scraping
=================================================
Here is a general guide on how to use your browser's Developer Tools
to ease the scraping process. Today almost all browsers come with
built in `Developer Tools`_ and although we will use Firefox in this
guide, the concepts are applicable to any other browser.
In this guide we'll introduce the basic tools to use from a browser's
Developer Tools by scraping `quotes.toscrape.com`_.
.. _topics-livedom:
Caveats with inspecting the live browser DOM
============================================
Since Developer Tools operate on a live browser DOM, what you'll actually see
when inspecting the page source is not the original HTML, but a modified one
after applying some browser clean up and executing Javascript code. Firefox,
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
the other hand, does not modify the original page HTML, so you won't be able to
extract any data if you use ``<tbody>`` in your XPath expressions.
Therefore, you should keep in mind the following things:
* Disable Javascript while inspecting the DOM looking for XPaths to be
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
* Never use full XPath paths, use relative and clever ones based on attributes
(such as ``id``, ``class``, ``width``, etc) or any identifying features like
``contains(@href, 'image')``.
* Never include ``<tbody>`` elements in your XPath expressions unless you
really know what you're doing
.. _topics-inspector:
Inspecting a website
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
`quotes.toscrape.com`_-site.
On the site we have a total of ten quotes from various authors with specific
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
Instead of viewing the whole source code for the page, we can simply right click
on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
In it you should see something like this:
.. image:: _images/inspector_01.png
:width: 777
:height: 469
:alt: Firefox's Inspector-tool
The interesting part for us is this:
.. code-block:: html
<div class="quote" itemscope="" itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">(...)</span>
<span>(...)</span>
<div class="tags">(...)</div>
</div>
If you hover over the first ``div`` directly above the ``span`` tag highlighted
in the screenshot, you'll see that the corresponding section of the webpage gets
highlighted as well. So now we have a section, but we can't find our quote text
anywhere.
The advantage of the `Inspector` is that it automatically expands and collapses
sections and tags of a webpage, which greatly improves readability. You can
expand and collapse a tag by clicking on the arrow in front of it or by double
clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
copy XPaths to selected elements. Let's try it out.
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
Then, back to your web browser, right-click on the ``span`` tag, select
``Copy > XPath`` and paste it in the Scrapy shell like so:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
see each quote:
.. code-block:: html
<div class="quote" itemscope="" itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
</span>
<span>(...)</span>
<div class="tags">(...)</div>
</div>
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
complex and by simply constructing an XPath with ``has-class("text")``
we were able to extract all quotes in one line.
The `Inspector` has a lot of other helpful features, such as searching in the
source code or directly scrolling to an element you selected. Let's demonstrate
a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
the need to find an element visually but the ``Scroll into View`` function
can be quite useful on complex sites.
Note that the search bar can also be used to search for and test CSS
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
.. _topics-network-tool:
The Network-tool
================
While scraping you may come across dynamic webpages where some parts
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
greatly facilitates this task. To demonstrate the Network-tool, let's
take a look at the page `quotes.toscrape.com/scroll`_.
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the Scrapy shell:
.. skip: next
.. code-block:: none
$ scrapy shell "quotes.toscrape.com/scroll"
(...)
>>> view(response)
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
.. image:: _images/network_01.png
:width: 777
:height: 296
:alt: Response from quotes.toscrape.com/scroll
The ``view(response)`` command let's us view the response our
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
the login-button and the footer, but the quotes are missing. This
tells us that the quotes are being loaded from a different request
than ``quotes.toscrape/scroll``.
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
log is automatically cleared each time you navigate to a different
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
If we reload the page now, you'll see the log get populated with six
new requests.
.. image:: _images/network_02.png
:width: 777
:height: 241
:alt: Network tab with persistent logs and requests
Here we see every request that has been made when reloading the page
and can inspect each request and its response. So let's find out
where our quotes are coming from:
First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address,
and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
.. image:: _images/network_03.png
:width: 777
:height: 375
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
import scrapy
import json
class QuoteSpider(scrapy.Spider):
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
page = 1
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
def parse(self, response):
data = json.loads(response.text)
for quote in data["quotes"]:
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
We iterate through the ``quotes`` and print out the ``quote["text"]``.
If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
into our ``url``.
.. _requests-from-curl:
In more complex websites, it could be difficult to easily reproduce the
requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
format, by right-clicking on each of them in the network tool and using the
:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
request::
from scrapy import Request
request = Request.from_curl(
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
"zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
"I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
.. _quotes.toscrape.com: http://quotes.toscrape.com
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions

253
docs/topics/dynamic-content.rst Normal file
View File

@ -0,0 +1,253 @@
.. _topics-dynamic-content:
====================================
Selecting dynamically-loaded content
====================================
Some webpages show the desired data when you load them in a web browser.
However, when you download them using Scrapy, you cannot reach the desired data
using :ref:`selectors <topics-selectors>`.
When this happens, the recommended approach is to
:ref:`find the data source <topics-finding-data-source>` and extract the data
from it.
If you fail to do that, and you can nonetheless access the desired data through
the :ref:`DOM <topics-livedom>` from your web browser, see
:ref:`topics-javascript-rendering`.
.. _topics-finding-data-source:
Finding the data source
=======================
To extract the desired data, you must first find its source location.
If the data is in a non-text-based format, such as an image or a PDF document,
use the :ref:`network tool <topics-network-tool>` of your web browser to find
the corresponding request, and :ref:`reproduce it
<topics-reproducing-requests>`.
If your web browser lets you select the desired data as text, the data may be
defined in embedded JavaScript code, or loaded from an external resource in a
text-based format.
In that case, you can use a tool like wgrep_ to find the URL of that resource.
If the data turns out to come from the original URL itself, you must
:ref:`inspect the source code of the webpage <topics-inspecting-source>` to
determine where the data is located.
If the data comes from a different URL, you will need to :ref:`reproduce the
corresponding request <topics-reproducing-requests>`.
.. _topics-inspecting-source:
Inspecting the source code of a webpage
=======================================
Sometimes you need to inspect the source code of a webpage (not the
:ref:`DOM <topics-livedom>`) to determine where some desired data is located.
Use Scrapy’s :command:`fetch` command to download the webpage contents as seen
by Scrapy::
scrapy fetch --nolog https://example.com > response.html
If the desired data is in embedded JavaScript code within a ``<script/>``
element, see :ref:`topics-parsing-javascript`.
If you cannot find the desired data, first make sure it’s not just Scrapy:
download the webpage with an HTTP client like curl_ or wget_ and see if the
information can be found in the response they get.
If they get a response with the desired data, modify your Scrapy
:class:`~scrapy.http.Request` to match that of the other HTTP client. For
example, try using the same user-agent string (:setting:`USER_AGENT`) or the
same :attr:`~scrapy.http.Request.headers`.
If they also get a response without the desired data, you’ll need to take
steps to make your request more similar to that of the web browser. See
:ref:`topics-reproducing-requests`.
.. _topics-reproducing-requests:
Reproducing requests
====================
Sometimes we need to reproduce a request the way our web browser performs it.
Use the :ref:`network tool <topics-network-tool>` of your web browser to see
how your web browser performs the desired request, and try to reproduce that
request with Scrapy.
It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
method and URL. However, you may also need to reproduce the body, headers and
form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
As all major browsers allow to export the requests in `cURL
<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
:class:`~scrapy.http.Request` from a cURL command. To get more information
visit :ref:`request from curl <requests-from-curl>` inside the network
tool section.
Once you get the expected response, you can :ref:`extract the desired data from
it <topics-handling-response-formats>`.
You can reproduce any request with Scrapy. However, some times reproducing all
necessary requests may not seem efficient in developer time. If that is your
case, and crawling speed is not a major concern for you, you can alternatively
consider :ref:`JavaScript pre-rendering <topics-javascript-rendering>`.
If you get the expected response `sometimes`, but not always, the issue is
probably not your request, but the target server. The target server might be
buggy, overloaded, or :ref:`banning <bans>` some of your requests.
.. _topics-handling-response-formats:
Handling different response formats
===================================
Once you have a response with the desired data, how you extract the desired
data from it depends on the type of response:
- If the response is HTML or XML, use :ref:`selectors
<topics-selectors>` as usual.
- If the response is JSON, use `json.loads`_ to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
data = json.loads(response.text)
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a
:class:`~scrapy.selector.Selector` and then
:ref:`use it <topics-selectors>` as usual::
selector = Selector(data['html'])
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
- If the response is CSS, use a `regular expression`_ to extract the desired
data from :attr:`response.text <scrapy.http.TextResponse.text>`.
.. _topics-parsing-images:
- If the response is an image or another format based on images (e.g. PDF),
read the response as bytes from
:attr:`response.body <scrapy.http.TextResponse.body>` and use an OCR
solution to extract the desired data as text.
For example, you can use pytesseract_. To read a table from a PDF,
`tabula-py`_ may be a better choice.
- If the response is SVG, or HTML with embedded SVG containing the desired
data, you may be able to extract the desired data using
:ref:`selectors <topics-selectors>`, since SVG is based on XML.
Otherwise, you might need to convert the SVG code into a raster image, and
:ref:`handle that raster image <topics-parsing-images>`.
.. _topics-parsing-javascript:
Parsing JavaScript code
=======================
If the desired data is hardcoded in JavaScript, you first need to get the
JavaScript code:
- If the JavaScript code is in a JavaScript file, simply read
:attr:`response.text <scrapy.http.TextResponse.text>`.
- If the JavaScript code is within a ``<script/>`` element of an HTML page,
use :ref:`selectors <topics-selectors>` to extract the text within that
``<script/>`` element.
Once you have a string with the JavaScript code, you can extract the desired
data from it:
- You might be able to use a `regular expression`_ to extract the desired
data in JSON format, which you can then parse with `json.loads`_.
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
>>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
>>> json_data = response.css('script::text').re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
For example, if the JavaScript code contains
``var data = {field: "value"};`` you can extract that data as follows:
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css('script::text').get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. _topics-javascript-rendering:
Pre-rendering JavaScript
========================
On webpages that fetch data from additional requests, reproducing those
requests that contain the desired data is the preferred approach. The effort is
often worth the result: structured, complete data with minimum parsing time and
network transfer.
However, sometimes it can be really hard to reproduce certain requests. Or you
may need something that no request can give you, such as a screenshot of a
webpage as seen in a web browser.
In these cases use the Splash_ JavaScript-rendering service, along with
`scrapy-splash`_ for seamless integration.
Splash returns as HTML the :ref:`DOM <topics-livedom>` of a webpage, so that
you can parse it with :ref:`selectors <topics-selectors>`. It provides great
flexibility through configuration_ or scripting_.
If you need something beyond what Splash offers, such as interacting with the
DOM on-the-fly from Python code instead of using a previously-written script,
or handling multiple web browser windows, you might need to
:ref:`use a headless browser <topics-headless-browsing>` instead.
.. _configuration: https://splash.readthedocs.io/en/stable/api.html
.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
.. _topics-headless-browsing:
Using a headless browser
========================
A `headless browser`_ is a special web browser that provides an API for
automation.
The easiest way to use a headless browser with Scrapy is to use Selenium_,
along with `scrapy-selenium`_ for seamless integration.
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
.. _curl: https://curl.haxx.se/
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _js2xml: https://github.com/scrapinghub/js2xml
.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
.. _pytesseract: https://github.com/madmaze/pytesseract
.. _regular expression: https://docs.python.org/3/library/re.html
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _Selenium: https://www.selenium.dev/
.. _Splash: https://github.com/scrapinghub/splash
.. _tabula-py: https://github.com/chezou/tabula-py
.. _wget: https://www.gnu.org/software/wget/
.. _wgrep: https://github.com/stav/wgrep

31
docs/topics/email.rst
View File

@ -9,19 +9,19 @@ Sending e-mail
Although Python makes sending e-mails relatively easy via the `smtplib`_
library, Scrapy provides its own facility for sending e-mails which is very
easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
interfering with the non-blocking IO of the crawler. It also provides a
simple API for sending attachments and it's very easy to configure, with a few
:ref:`settings <topics-email-settings>`.
easy to use and it's implemented using :doc:`Twisted non-blocking IO
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
IO of the crawler. It also provides a simple API for sending attachments and
it's very easy to configure, with a few :ref:`settings
<topics-email-settings>`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
the standard constructor::
the standard ``__init__`` method::
from scrapy.mail import MailSender
mailer = MailSender()
@ -35,17 +35,12 @@ And here is how to use it to send an e-mail (without attachments)::
mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
.. note::
As shown in the example above, ``to`` and ``cc`` need to be lists
of email addresses, not single addresses, and even for one recipient,
i.e. ``to="someone@example.com"`` will not work.
MailSender class reference
==========================
MailSender is the preferred class to use for sending emails from Scrapy, as it
uses `Twisted non-blocking IO`_, like the rest of the framework.
uses :doc:`Twisted non-blocking IO <twisted:core/howto/defer-intro>`, like the
rest of the framework.
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
@ -60,10 +55,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
:param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER`
setting will be used. If not given, no SMTP authentication will be
performed.
:type smtphost: str
:type smtphost: str or bytes
:param smtppass: the SMTP pass for authentication.
:type smtppass: str
:type smtppass: str or bytes
:param smtpport: the SMTP port to connect to
:type smtpport: int
@ -87,13 +82,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
Send email to the given recipients.
:param to: the e-mail recipients
:type to: list
:type to: str or list of str
:param subject: the subject of the e-mail
:type subject: str
:param cc: the e-mails to CC
:type cc: list
:type cc: str or list of str
:param body: the e-mail body
:type body: str
@ -117,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
Mail settings
=============
These settings define the default constructor values of the :class:`MailSender`
These settings define the default ``__init__`` method values of the :class:`MailSender`
class, and can be used to configure e-mail notifications in your project without
writing any code (for those extensions and code that uses :class:`MailSender`).

8
docs/topics/exceptions.rst
View File

@ -39,6 +39,14 @@ For example::
if 'Bandwidth exceeded' in response.body:
raise CloseSpider('bandwidth_exceeded')
DontCloseSpider
---------------
.. exception:: DontCloseSpider
This exception can be raised in a :signal:`spider_idle` signal handler to
prevent the spider from being closed.
IgnoreRequest
-------------

138
docs/topics/exporters.rst
View File

@ -36,38 +36,35 @@ to export
3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
the end of the exporting process
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
Exporter to export scraped items to different files, one per spider::
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
from scrapy import signals
from scrapy.exporters import XmlItemExporter
from scrapy.exporters import XmlItemExporter
class XmlExportPipeline(object):
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
def __init__(self):
self.files = {}
def open_spider(self, spider):
self.year_to_exporter = {}
@classmethod
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
return pipeline
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
exporter.finish_exporting()
def spider_opened(self, spider):
file = open('%s_products.xml' % spider.name, 'w+b')
self.files[spider] = file
self.exporter = XmlItemExporter(file)
self.exporter.start_exporting()
def _exporter_for_item(self, item):
year = item['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
exporter = XmlItemExporter(f)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
return self.year_to_exporter[year]
def spider_closed(self, spider):
self.exporter.finish_exporting()
file = self.files.pop(spider)
file.close()
def process_item(self, item, spider):
self.exporter.export_item(item)
return item
def process_item(self, item, spider):
exporter = self._exporter_for_item(item)
exporter.export_item(item)
return item
.. _topics-exporters-field-serialization:
@ -90,8 +87,8 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
If you use :class:`~.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
If you use :class:`~.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
@ -140,16 +137,19 @@ output examples, which assume you're exporting these two items::
BaseItemExporter
----------------
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8')
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
defining what fields to export, whether to export empty fields, or which
encoding to use.
These features can be configured through the constructor arguments which
These features can be configured through the ``__init__`` method arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`.
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
.. versionadded:: 2.0
The *dont_fail* parameter.
.. method:: export_item(item)
@ -167,9 +167,9 @@ BaseItemExporter
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
:param field: the field being serialized. If a raw dict is being
:param field: the field being serialized. If a raw dict is being
exported (not :class:`~.Item`) *field* value is an empty dict.
:type field: :class:`~scrapy.item.Field` object or an empty dict
:type field: :class:`~scrapy.item.Field` object or an empty dict
:param name: the name of the field being serialized
:type name: str
@ -216,6 +216,21 @@ BaseItemExporter
encoding). Other value types are passed unchanged to the specific
serialization library.
.. attribute:: indent
Amount of spaces used to indent the output on each level. Defaults to ``0``.
* ``indent=None`` selects the most compact representation,
all items in the same line with no indentation
* ``indent<=0`` each item on its own line, no indentation
* ``indent>0`` each item on its own line, indented with the provided numeric value
PythonItemExporter
------------------
.. autoclass:: PythonItemExporter
.. highlight:: none
XmlItemExporter
@ -225,7 +240,8 @@ XmlItemExporter
Exports Items in XML format to the specified file object.
:param file: the file-like object to use for exporting the data.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
:param root_element: The name of root element in the exported XML.
:type root_element: str
@ -233,8 +249,8 @@ XmlItemExporter
:param item_element: The name of each item element in the exported XML.
:type item_element: str
The additional keyword arguments of this constructor are passed to the
:class:`BaseItemExporter` constructor.
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@ -281,7 +297,8 @@ CsvItemExporter
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
:param file: the file-like object to use for exporting the data.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
:param include_headers_line: If enabled, makes the exporter output a header
line with the field names taken from
@ -292,9 +309,9 @@ CsvItemExporter
multi-valued fields, if found.
:type include_headers_line: str
The additional keyword arguments of this constructor are passed to the
:class:`BaseItemExporter` constructor, and the leftover arguments to the
`csv.writer`_ constructor, so you can use any `csv.writer` constructor
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
argument to customize this exporter.
A typical output of this exporter would be::
@ -312,15 +329,16 @@ PickleItemExporter
Exports Items in pickle format to the given file-like object.
:param file: the file-like object to use for exporting the data.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
:param protocol: The pickle protocol to use.
:type protocol: int
For more information, refer to the `pickle module documentation`_.
The additional keyword arguments of this constructor are passed to the
:class:`BaseItemExporter` constructor.
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
@ -333,10 +351,11 @@ PprintItemExporter
Exports Items in pretty print format to the specified file object.
:param file: the file-like object to use for exporting the data.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
The additional keyword arguments of this constructor are passed to the
:class:`BaseItemExporter` constructor.
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@ -351,12 +370,13 @@ JsonItemExporter
.. class:: JsonItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing all
objects as a list of objects. The additional constructor arguments are
passed to the :class:`BaseItemExporter` constructor, and the leftover
arguments to the `JSONEncoder`_ constructor, so you can use any
`JSONEncoder`_ constructor argument to customize this exporter.
objects as a list of objects. The additional ``__init__`` method arguments are
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
A typical output of this exporter would be::
@ -381,12 +401,13 @@ JsonLinesItemExporter
.. class:: JsonLinesItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing one
JSON-encoded item per line. The additional constructor arguments are passed
to the :class:`BaseItemExporter` constructor, and the leftover arguments to
the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
constructor argument to customize this exporter.
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
A typical output of this exporter would be::
@ -397,3 +418,8 @@ JsonLinesItemExporter
this exporter is well suited for serializing large amounts of data.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
MarshalItemExporter
-------------------
.. autoclass:: MarshalItemExporter

39
docs/topics/extensions.rst
View File

@ -19,7 +19,7 @@ settings, just like any other Scrapy code.
It is customary for extensions to prefix their settings with their own name, to
avoid collision with existing (and future) extensions. For example, a
hypothetic extension to handle `Google Sitemaps`_ would use settings like
`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
@ -28,7 +28,7 @@ Loading & activating extensions
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
code must be performed in the class constructor (``__init__`` method).
code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set.
Disabling an extension
======================
In order to disable an extension that comes enabled by default (ie. those
In order to disable an extension that comes enabled by default (i.e. those
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
``None``. For example::
@ -107,7 +107,7 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging(object):
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count
@ -183,7 +183,7 @@ Telnet console extension
.. module:: scrapy.extensions.telnet
:synopsis: Telnet console
.. class:: scrapy.extensions.telnet.TelnetConsole
.. class:: TelnetConsole
Provides a telnet console for getting into a Python interpreter inside the
currently running Scrapy process, which can be very useful for debugging.
@ -200,7 +200,7 @@ Memory usage extension
.. module:: scrapy.extensions.memusage
:synopsis: Memory usage extension
.. class:: scrapy.extensions.memusage.MemoryUsage
.. class:: MemoryUsage
.. note:: This extension does not work in Windows.
@ -220,7 +220,6 @@ can be configured with the following settings:
* :setting:`MEMUSAGE_LIMIT_MB`
* :setting:`MEMUSAGE_WARNING_MB`
* :setting:`MEMUSAGE_NOTIFY_MAIL`
* :setting:`MEMUSAGE_REPORT`
* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS`
Memory debugger extension
@ -229,7 +228,7 @@ Memory debugger extension
.. module:: scrapy.extensions.memdebug
:synopsis: Memory debugger extension
.. class:: scrapy.extensions.memdebug.MemoryDebugger
.. class:: MemoryDebugger
An extension for debugging memory usage. It collects information about:
@ -245,7 +244,7 @@ Close spider extension
.. module:: scrapy.extensions.closespider
:synopsis: Close spider extension
.. class:: scrapy.extensions.closespider.CloseSpider
.. class:: CloseSpider
Closes a spider automatically when some conditions are met, using a specific
closing reason for each condition.
@ -278,9 +277,11 @@ CLOSESPIDER_ITEMCOUNT
Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount if items and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``. If zero (or
non set), spiders won't be closed by number of passed items.
that amount and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``.
Requests which are currently in the downloader queue (up to
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
If zero (or non set), spiders won't be closed by number of passed items.
.. setting:: CLOSESPIDER_PAGECOUNT
@ -316,7 +317,7 @@ StatsMailer extension
.. module:: scrapy.extensions.statsmailer
:synopsis: StatsMailer extension
.. class:: scrapy.extensions.statsmailer.StatsMailer
.. class:: StatsMailer
This simple extension can be used to send a notification e-mail every time a
domain has finished scraping, including the Scrapy stats collected. The email
@ -332,7 +333,7 @@ Debugging extensions
Stack trace dump extension
~~~~~~~~~~~~~~~~~~~~~~~~~~
.. class:: scrapy.extensions.debug.StackTraceDump
.. class:: StackTraceDump
Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_
signal is received. The information dumped is the following:
@ -344,7 +345,7 @@ signal is received. The information dumped is the following:
After the stack trace and engine status is dumped, the Scrapy process continues
running normally.
This extension only works on POSIX-compliant platforms (ie. not Windows),
This extension only works on POSIX-compliant platforms (i.e. not Windows),
because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows.
There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
@ -361,15 +362,15 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
Debugger extension
~~~~~~~~~~~~~~~~~~
.. class:: scrapy.extensions.debug.Debugger
.. class:: Debugger
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
signal is received. After the debugger is exited, the Scrapy process continues
running normally.
For more info see `Debugging in Python`.
For more info see `Debugging in Python`_.
This extension only works on POSIX-compliant platforms (ie. not Windows).
This extension only works on POSIX-compliant platforms (i.e. not Windows).
.. _Python debugger: https://docs.python.org/2/library/pdb.html
.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

144
docs/topics/feed-exports.rst
View File

@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "export
feed") to be consumed by other systems.
Scrapy provides this functionality out of the box with the Feed Exports, which
allows you to generate a feed with the scraped items, using multiple
allows you to generate feeds with the scraped items, using multiple
serialization formats and storage backends.
.. _topics-feed-format:
@ -36,7 +36,7 @@ But you can also extend the supported format through the
JSON
----
* :setting:`FEED_FORMAT`: ``json``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
@ -46,7 +46,7 @@ JSON
JSON lines
----------
* :setting:`FEED_FORMAT`: ``jsonlines``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
.. _topics-feed-format-csv:
@ -54,7 +54,7 @@ JSON lines
CSV
---
* :setting:`FEED_FORMAT`: ``csv``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
@ -66,7 +66,7 @@ CSV
XML
---
* :setting:`FEED_FORMAT`: ``xml``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
.. _topics-feed-format-pickle:
@ -74,7 +74,7 @@ XML
Pickle
------
* :setting:`FEED_FORMAT`: ``pickle``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
.. _topics-feed-format-marshal:
@ -82,7 +82,7 @@ Pickle
Marshal
-------
* :setting:`FEED_FORMAT`: ``marshal``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
@ -91,20 +91,20 @@ Marshal
Storages
========
When using the feed exports you define where to store the feed using a URI_
(through the :setting:`FEED_URI` setting). The feed exports supports multiple
When using the feed exports you define where to store the feed using one or multiple URIs_
(through the :setting:`FEEDS` setting). The feed exports supports multiple
storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
or boto_ library is installed (Scrapy supports boto_ only on Python 2).
library is installed.
.. _topics-feed-uri-params:
@ -164,6 +164,11 @@ The feeds are stored in a FTP server.
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
FTP supports two different connection modes: `active or passive
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
.. _topics-feed-storage-s3:
S3
@ -177,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
* Required external libraries: `botocore`_ or `boto`_
* Required external libraries: `botocore`_
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@ -185,6 +190,10 @@ passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
You can also define a custom ACL for exported feeds using this setting:
* :setting:`FEED_STORAGE_S3_ACL`
.. _topics-feed-storage-stdout:
Standard output
@ -202,35 +211,66 @@ Settings
These are the settings used for configuring the feed exports:
* :setting:`FEED_URI` (mandatory)
* :setting:`FEED_FORMAT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEEDS` (mandatory)
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
.. currentmodule:: scrapy.extensions.feedexport
.. setting:: FEED_URI
.. setting:: FEEDS
FEED_URI
--------
FEEDS
-----
Default: ``None``
.. versionadded:: 2.1
The URI of the export feed. See :ref:`topics-feed-storage-backends` for
supported URI schemes.
Default: ``{}``
This setting is required for enabling the feed exports.
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
This setting is required for enabling the feed export feature.
.. setting:: FEED_FORMAT
See :ref:`topics-feed-storage-backends` for supported URI schemes.
FEED_FORMAT
-----------
For instance::
The serialization format to be used for the feed. See
:ref:`topics-feed-format` for possible values.
{
'items.json': {
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
'fields': None,
'indent': 4,
},
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],
'encoding': 'latin1',
'indent': 8,
},
pathlib.Path('items.csv'): {
'format': 'csv',
'fields': ['price', 'name'],
},
}
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition.
* ``format``: the serialization format to be used for the feed.
See :ref:`topics-feed-format` for possible values.
Mandatory, no fallback setting
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
.. setting:: FEED_EXPORT_ENCODING
@ -266,6 +306,22 @@ If an exporter requires a fixed set of fields (this is the case for
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
.. setting:: FEED_EXPORT_INDENT
FEED_EXPORT_INDENT
------------------
Default: ``0``
Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
is a non-negative integer, then array elements and object members will be pretty-printed
with that indent level. An indent level of ``0`` (the default), or negative,
will put each item on a new line. ``None`` selects the most compact representation.
Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter`
and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exporting
to ``.json`` or ``.xml``.
.. setting:: FEED_STORE_EMPTY
FEED_STORE_EMPTY
@ -273,7 +329,7 @@ FEED_STORE_EMPTY
Default: ``False``
Whether to export empty feeds (ie. feeds with no items).
Whether to export empty feeds (i.e. feeds with no items).
.. setting:: FEED_STORAGES
@ -285,6 +341,30 @@ Default: ``{}``
A dict containing additional feed storage backends supported by your project.
The keys are URI schemes and the values are paths to storage classes.
.. setting:: FEED_STORAGE_FTP_ACTIVE
FEED_STORAGE_FTP_ACTIVE
-----------------------
Default: ``False``
Whether to use the active connection mode when exporting feeds to an FTP server
(``True``) or use the passive connection mode instead (``False``, default).
For information about FTP connection modes, see `What is the difference between
active and passive FTP? <https://stackoverflow.com/a/1699163>`_.
.. setting:: FEED_STORAGE_S3_ACL
FEED_STORAGE_S3_ACL
-------------------
Default: ``''`` (empty string)
A string containing a custom ACL for feeds exported to Amazon S3 by your project.
For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs.
.. setting:: FEED_STORAGES_BASE
FEED_STORAGES_BASE
@ -345,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
'csv': None,
}
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _boto: https://github.com/boto/boto
.. _botocore: https://github.com/boto/botocore
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl

167
docs/topics/firebug.rst
View File

@ -1,167 +0,0 @@
.. _topics-firebug:
==========================
Using Firebug for scraping
==========================
.. note:: Google Directory, the example website used in this guide is no longer
available as it `has been shut down by Google`_. The concepts in this guide
are still valid though. If you want to update this guide to use a new
(working) site, your contribution will be more than welcome!. See :ref:`topics-contributing`
for information on how to do so.
Introduction
============
This document explains how to use `Firebug`_ (a Firefox add-on) to make the
scraping process easier and more fun. For other useful Firefox add-ons see
:ref:`topics-firefox-addons`. There are some caveats with using Firefox add-ons
to inspect pages, see :ref:`topics-firefox-livedom`.
In this example, we'll show how to use `Firebug`_ to scrape data from the
`Google Directory`_, which contains the same data as the `Open Directory
Project`_ used in the :ref:`tutorial <intro-tutorial>` but with a different
face.
.. _Firebug: http://getfirebug.com
.. _Google Directory: http://directory.google.com/
.. _Open Directory Project: http://www.dmoz.org
Firebug comes with a very useful feature called `Inspect Element`_ which allows
you to inspect the HTML code of the different page elements just by hovering
your mouse over them. Otherwise you would have to search for the tags manually
through the HTML body which can be a very tedious task.
.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
In the following screenshot you can see the `Inspect Element`_ tool in action.
.. image:: _images/firebug1.png
:width: 913
:height: 600
:alt: Inspecting elements with Firebug
At first sight, we can see that the directory is divided in categories, which
are also divided in subcategories.
However, it seems that there are more subcategories than the ones being shown
in this page, so we'll keep looking:
.. image:: _images/firebug2.png
:width: 819
:height: 629
:alt: Inspecting elements with Firebug
As expected, the subcategories contain links to other subcategories, and also
links to actual websites, which is the purpose of the directory.
Getting links to follow
=======================
By looking at the category URLs we can see they share a pattern:
http://directory.google.com/Category/Subcategory/Another_Subcategory
Once we know that, we are able to construct a regular expression to follow
those links. For example, the following one::
directory\.google\.com/[A-Z][a-zA-Z_/]+$
So, based on that regular expression we can create the first crawling rule::
Rule(LinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$', ),
'parse_category',
follow=True,
),
The :class:`~scrapy.spiders.Rule` object instructs
:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the
category links. ``parse_category`` will be a method of the spider which will
process and extract data from those pages.
This is how the spider would look so far::
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class GoogleDirectorySpider(CrawlSpider):
name = 'directory.google.com'
allowed_domains = ['directory.google.com']
start_urls = ['http://directory.google.com/']
rules = (
Rule(LinkExtractor(allow='directory\.google\.com/[A-Z][a-zA-Z_/]+$'),
'parse_category', follow=True,
),
)
def parse_category(self, response):
# write the category page data extraction code here
pass
Extracting the data
===================
Now we're going to write the code to extract data from those pages.
With the help of Firebug, we'll take a look at some page containing links to
websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can
extract those links using :ref:`Selectors <topics-selectors>`. We'll also
use the :ref:`Scrapy shell <topics-shell>` to test those XPath's and make sure
they work as we expect.
.. image:: _images/firebug3.png
:width: 965
:height: 751
:alt: Inspecting elements with Firebug
As you can see, the page markup is not very descriptive: the elements don't
contain ``id``, ``class`` or any attribute that clearly identifies them, so
we'll use the ranking bars as a reference point to select the data to extract
when we construct our XPaths.
After using FireBug, we can see that each link is inside a ``td`` tag, which is
itself inside a ``tr`` tag that also contains the link's ranking bar (in
another ``td``).
So we can select the ranking bar, then find its parent (the ``tr``), and then
finally, the link's ``td`` (which contains the data we want to scrape).
This results in the following XPath::
//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td//a
It's important to use the :ref:`Scrapy shell <topics-shell>` to test these
complex XPath expressions and make sure they work as expected.
Basically, that expression will look for the ranking bar's ``td`` element, and
then select any ``td`` element who has a descendant ``a`` element whose
``href`` attribute contains the string ``#pagerank``"
Of course, this is not the only XPath, and maybe not the simpler one to select
that data. Another approach could be, for example, to find any ``font`` tags
that have that grey colour of the links,
Finally, we can write our ``parse_category()`` method::
def parse_category(self, response):
# The path to website links in directory page
links = response.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
for link in links:
item = DirectoryItem()
item['name'] = link.xpath('a/text()').extract()
item['url'] = link.xpath('a/@href').extract()
item['description'] = link.xpath('font[2]/text()').extract()
yield item
Be aware that you may find some elements which appear in Firebug but
not in the original HTML, such as the typical case of ``<tbody>``
elements.
or tags which Therefer in page HTML
sources may on Firebug inspects the live DOM
.. _has been shut down by Google: https://searchenginewatch.com/sew/news/2096661/google-directory-shut

82
docs/topics/firefox.rst
View File

@ -1,82 +0,0 @@
.. _topics-firefox:
==========================
Using Firefox for scraping
==========================
Here is a list of tips and advice on using Firefox for scraping, along with a
list of useful Firefox add-ons to ease the scraping process.
.. _topics-firefox-livedom:
Caveats with inspecting the live browser DOM
============================================
Since Firefox add-ons operate on a live browser DOM, what you'll actually see
when inspecting the page source is not the original HTML, but a modified one
after applying some browser clean up and executing Javascript code. Firefox,
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
the other hand, does not modify the original page HTML, so you won't be able to
extract any data if you use ``<tbody>`` in your XPath expressions.
Therefore, you should keep in mind the following things when working with
Firefox and XPath:
* Disable Firefox Javascript while inspecting the DOM looking for XPaths to be
used in Scrapy
* Never use full XPath paths, use relative and clever ones based on attributes
(such as ``id``, ``class``, ``width``, etc) or any identifying features like
``contains(@href, 'image')``.
* Never include ``<tbody>`` elements in your XPath expressions unless you
really know what you're doing
.. _topics-firefox-addons:
Useful Firefox add-ons for scraping
===================================
Firebug
-------
`Firebug`_ is a widely known tool among web developers and it's also very
useful for scraping. In particular, its `Inspect Element`_ feature comes very
handy when you need to construct the XPaths for extracting data because it
allows you to view the HTML code of each page element while moving your mouse
over it.
See :ref:`topics-firebug` for a detailed guide on how to use Firebug with
Scrapy.
XPather
-------
`XPather`_ allows you to test XPath expressions directly on the pages.
XPath Checker
-------------
`XPath Checker`_ is another Firefox add-on for testing XPaths on your pages.
Tamper Data
-----------
`Tamper Data`_ is a Firefox add-on which allows you to view and modify the HTTP
request headers sent by Firefox. Firebug also allows to view HTTP headers, but
not to modify them.
Firecookie
----------
`Firecookie`_ makes it easier to view and manage cookies. You can use this
extension to create a new cookie, delete existing cookies, see a list of cookies
for the current site, manage cookies permissions and a lot more.
.. _Firebug: http://getfirebug.com
.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/

49
docs/topics/item-pipeline.rst
View File

@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
(or any descendant class) object, return a `Twisted Deferred`_ or raise
(or any descendant class) object, return a
:class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
Item pipeline example
=====================
@ -82,13 +81,13 @@ contain a price::
from scrapy.exceptions import DropItem
class PricePipeline(object):
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
if item['price']:
if item['price_excludes_vat']:
if item.get('price'):
if item.get('price_excludes_vat'):
item['price'] = item['price'] * self.vat_factor
return item
else:
@ -104,10 +103,10 @@ format::
import json
class JsonWriterPipeline(object):
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'wb')
self.file = open('items.jl', 'w')
def close_spider(self, spider):
self.file.close()
@ -133,8 +132,8 @@ method and how to clean up the resources properly.::
import pymongo
class MongoPipeline(object):
class MongoPipeline:
collection_name = 'scrapy_items'
def __init__(self, mongo_uri, mongo_db):
@ -156,20 +155,23 @@ method and how to clean up the resources properly.::
self.client.close()
def process_item(self, item, spider):
self.db[self.collection_name].insert(dict(item))
self.db[self.collection_name].insert_one(dict(item))
return item
.. _MongoDB: https://www.mongodb.org/
.. _pymongo: https://api.mongodb.org/python/current/
.. _MongoDB: https://www.mongodb.com/
.. _pymongo: https://api.mongodb.com/python/current/
.. _ScreenshotPipeline:
Take screenshot of item
-----------------------
This example demonstrates how to return Deferred_ from :meth:`process_item` method.
This example demonstrates how to return a
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded
and Deferred callback fires, it saves item to a file and adds filename to an item.
makes request to locally running instance of Splash_. After request is downloaded,
it saves the screenshot to a file and adds filename to the item.
::
@ -178,21 +180,18 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
from urllib.parse import quote
class ScreenshotPipeline(object):
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
every Scrapy item."""
SPLASH_URL = "http://localhost:8050/render.png?url={}"
def process_item(self, item, spider):
async def process_item(self, item, spider):
encoded_item_url = quote(item["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
dfd = spider.crawler.engine.download(request, spider)
dfd.addBoth(self.return_item, item)
return dfd
response = await spider.crawler.engine.download(request, spider)
def return_item(self, response, item):
if response.status != 200:
# Error happened, return item.
return item
@ -208,8 +207,7 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
item["screenshot_filename"] = filename
return item
.. _Splash: http://splash.readthedocs.io/en/stable/
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
.. _Splash: https://splash.readthedocs.io/en/stable/
Duplicates filter
-----------------
@ -221,7 +219,7 @@ returns multiples items with the same id::
from scrapy.exceptions import DropItem
class DuplicatesPipeline(object):
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()
@ -248,4 +246,3 @@ To activate an Item Pipeline component you must add its class to the
The integer values you assign to classes in this setting determine the
order in which they run: items go through from lower valued to higher
valued classes. It's customary to define these numbers in the 0-1000 range.

170
docs/topics/items.rst
View File

@ -16,12 +16,12 @@ especially in a larger project with many spiders.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
their available fields.
their available fields.
Various Scrapy components use extra information provided by Items:
Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
tracks Item instances to help finding memory leaks
tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
@ -40,6 +40,7 @@ objects. Here is an example::
name = scrapy.Field()
price = scrapy.Field()
stock = scrapy.Field()
tags = scrapy.Field()
last_updated = scrapy.Field(serializer=str)
.. note:: Those familiar with `Django`_ will notice that Scrapy Items are
@ -83,105 +84,128 @@ notice the API is very similar to the `dict API`_.
Creating items
--------------
::
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
>>> product = Product(name='Desktop PC', price=1000)
>>> print product
Product(name='Desktop PC', price=1000)
Getting field values
--------------------
::
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
>>> product['price']
1000
>>> product['price']
1000
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product.get('last_updated', 'not set')
not set
>>> product.get('last_updated', 'not set')
not set
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product.get('lala', 'unknown field')
'unknown field'
>>> product.get('lala', 'unknown field')
'unknown field'
>>> 'name' in product # is name field populated?
True
>>> 'name' in product # is name field populated?
True
>>> 'last_updated' in product # is last_updated populated?
False
>>> 'last_updated' in product # is last_updated populated?
False
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> 'lala' in product.fields # is lala a declared field?
False
>>> 'lala' in product.fields # is lala a declared field?
False
Setting field values
--------------------
::
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
To access all populated values, just use the typical `dict API`_::
To access all populated values, just use the typical `dict API`_:
>>> product.keys()
['price', 'name']
>>> product.keys()
['price', 'name']
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
.. _copying-items:
Copying items
-------------
To copy an item, you must first decide whether you want a shallow copy or a
deep copy.
If your item contains mutable_ values like lists or dictionaries, a shallow
copy will keep references to the same mutable values across all different
copies.
.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
For example, if you have an item with a list of tags, and you create a shallow
copy of that item, both the original item and the copy have the same list of
tags. Adding a tag to the list of one of the items will add the tag to the
other item as well.
If that is not the desired behavior, use a deep copy instead.
See the `documentation of the copy module`_ for more information.
.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
To create a shallow copy of an item, you can either call
:meth:`~scrapy.item.Item.copy` on an existing item
(``product2 = product.copy()``) or instantiate your item class from an existing
item (``product2 = Product(product)``).
To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
(``product2 = product.deepcopy()``).
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
Other common tasks
------------------
Copying items::
Creating dicts from items:
>>> product2 = Product(product)
>>> print product2
Product(name='Desktop PC', price=1000)
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
>>> product3 = product2.copy()
>>> print product3
Product(name='Desktop PC', price=1000)
Creating items from dicts:
Creating dicts from items::
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Creating items from dicts::
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Extending Items
===============
@ -211,8 +235,12 @@ Item objects
Return a new Item optionally initialized from the given argument.
Items replicate the standard `dict API`_, including its constructor. The
only additional attribute provided by Items is:
Items replicate the standard `dict API`_, including its ``__init__`` method, and
also provide the following additional API members:
.. automethod:: copy
.. automethod:: deepcopy
.. attribute:: fields
@ -237,3 +265,9 @@ Field objects
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
Other classes related to Item
=============================
.. autoclass:: BaseItem
.. autoclass:: ItemMeta

37
docs/topics/jobs.rst
View File

@ -22,7 +22,7 @@ Job directory
To enable persistence support you just need to define a *job directory* through
the ``JOBDIR`` setting. This directory will be for storing all required data to
keep the state of a single job (ie. a spider run). It's important to note that
keep the state of a single job (i.e. a spider run). It's important to note that
this directory must not be shared by different spiders, or even different
jobs/runs of the same spider, as it's meant to be used for storing the state of
a *single* job.
@ -30,7 +30,7 @@ a *single* job.
How to use it
=============
To start a spider with persistence supported enabled, run it like this::
To start a spider with persistence support enabled, run it like this::
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
@ -68,36 +68,17 @@ Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
on cookies.
.. _request-serialization:
Request serialization
---------------------
Requests must be serializable by the `pickle` module, in order for persistence
to work, so you should make sure that your requests are serializable.
The most common issue here is to use ``lambda`` functions on request callbacks that
can't be persisted.
So, for example, this won't work::
def some_callback(self, response):
somearg = 'test'
return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
def other_callback(self, response, somearg):
print "the argument passed is:", somearg
But this will::
def some_callback(self, response):
somearg = 'test'
return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
def other_callback(self, response):
somearg = response.meta['somearg']
print "the argument passed is:", somearg
For persistence to work, :class:`~scrapy.http.Request` objects must be
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
values passed to their ``__init__`` method, which must be methods of the
running :class:`~scrapy.spiders.Spider` class.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
It is ``False`` by default.
.. _pickle: http://docs.python.org/library/pickle.html

137
docs/topics/leaks.rst
View File

@ -17,8 +17,8 @@ what is known as a "memory leak".
To help debugging memory leaks, Scrapy provides a built-in mechanism for
tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
and you can also use a third-party library called :ref:`Guppy
<topics-leaks-guppy>` for more advanced memory debugging (see below for more
and you can also use a third-party library called :ref:`muppy
<topics-leaks-muppy>` for more advanced memory debugging (see below for more
info). Both mechanisms must be used from the :ref:`Telnet Console
<topics-telnetconsole>`.
@ -27,10 +27,11 @@ Common causes of memory leaks
It happens quite often (sometimes by accident, sometimes on purpose) that the
Scrapy developer passes objects referenced in Requests (for example, using the
:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
and that effectively bounds the lifetime of those referenced objects to the
lifetime of the Request. This is, by far, the most common cause of memory leaks
in Scrapy projects, and a quite difficult one to debug for newcomers.
:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
attributes or the request callback function) and that effectively bounds the
lifetime of those referenced objects to the lifetime of the Request. This is,
by far, the most common cause of memory leaks in Scrapy projects, and a quite
difficult one to debug for newcomers.
In big projects, the spiders are typically written by different people and some
of those spiders could be "leaking" and thus affecting the rest of the other
@ -48,7 +49,8 @@ Too Many Requests?
By default Scrapy keeps the request queue in memory; it includes
:class:`~scrapy.http.Request` objects and all objects
referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
and :attr:`~scrapy.http.Request.meta`).
While not necessarily a leak, this can take a lot of memory. Enabling
:ref:`persistent job queue <topics-jobs>` could help keeping memory usage
in control.
@ -101,14 +103,14 @@ Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
callback=self.parse, meta={referer: response})
callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
ties the response lifetime to the requests' one, and that would definitely
cause memory leaks.
Let's see how we can discover the cause (without knowing it
a-priori, of course) by using the ``trackref`` tool.
a priori, of course) by using the ``trackref`` tool.
After the crawler is running for a few minutes and we notice its memory usage
has grown a lot, we can enter its telnet console and check the live
@ -130,21 +132,21 @@ and check the code of the spider to discover the nasty line that is
generating the leaks (passing response references inside requests).
Sometimes extra information about live objects can be helpful.
Let's check the oldest response::
Let's check the oldest response:
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
If you want to iterate over all objects, instead of getting the oldest one, you
can use the :func:`scrapy.utils.trackref.iter_all` function::
can use the :func:`scrapy.utils.trackref.iter_all` function:
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
Too many spiders?
-----------------
@ -153,10 +155,10 @@ If your project has too many spiders executed in parallel,
the output of :func:`prefs()` can be difficult to read.
For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclases). For
example, this won't show any live references to spiders::
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. module:: scrapy.utils.trackref
:synopsis: Track references of live objects
@ -168,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
.. class:: object_ref
Inherit from this class (instead of object) if you want to track live
Inherit from this class if you want to track live
instances with the ``trackref`` module.
.. function:: print_live_refs(class_name, ignore=NoneType)
@ -191,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
``None`` if none is found. Use :func:`print_live_refs` first to get a list
of all tracked live objects per class name.
.. _topics-leaks-guppy:
.. _topics-leaks-muppy:
Debugging memory leaks with Guppy
Debugging memory leaks with muppy
=================================
``trackref`` provides a very convenient mechanism for tracking down memory
@ -201,57 +203,48 @@ leaks, but it only keeps track of the objects that are more likely to cause
memory leaks (Requests, Responses, Items, and Selectors). However, there are
other cases where the memory leaks could come from other (more or less obscure)
objects. If this is your case, and you can't find your leaks using ``trackref``,
you still have another resource: the `Guppy library`_.
you still have another resource: the muppy library.
.. _Guppy library: https://pypi.python.org/pypi/guppy
If you use ``pip``, you can install Guppy with the following command::
You can use muppy from `Pympler`_.
pip install guppy
.. _Pympler: https://pypi.org/project/Pympler/
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
Guppy heap objects. Here's an example to view all Python objects available in
the heap using Guppy::
If you use ``pip``, you can install muppy with the following command::
>>> x = hpy.heap()
>>> x.bytype
Partition of a set of 297033 objects. Total size = 52587824 bytes.
Index Count % Size % Cumulative % Type
0 22307 8 16423880 31 16423880 31 dict
1 122285 41 12441544 24 28865424 55 str
2 68346 23 5966696 11 34832120 66 tuple
3 227 0 5836528 11 40668648 77 unicode
4 2461 1 2222272 4 42890920 82 type
5 16870 6 2024400 4 44915320 85 function
6 13949 5 1673880 3 46589200 89 types.CodeType
7 13422 5 1653104 3 48242304 92 list
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
<1676 more rows. Type e.g. '_.more' to view.>
pip install Pympler
You can see that most space is used by dicts. Then, if you want to see from
which attribute those dicts are referenced, you could do::
Here's an example to view all Python objects available in
the heap using muppy:
>>> x.bytype[0].byvia
Partition of a set of 22307 objects. Total size = 16423880 bytes.
Index Count % Size % Cumulative % Referred Via:
0 10982 49 9416336 57 9416336 57 '.__dict__'
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
2 3097 14 1122904 7 13220744 80
3 990 4 277200 2 13497944 82 "['cookies']"
4 987 4 276360 2 13774304 84 "['cache']"
5 985 4 275800 2 14050104 86 "['meta']"
6 897 4 251160 2 14301264 87 '[2]'
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
9 27 0 155016 1 14841328 90 '[1]'
<333 more rows. Type e.g. '_.more' to view.>
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
As you can see, the Guppy module is very powerful but also requires some deep
knowledge about Python internals. For more info about Guppy, refer to the
`Guppy documentation`_.
For more info about muppy, refer to the `muppy documentation`_.
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
.. _muppy documentation: https://pythonhosted.org/Pympler/muppy.html
.. _topics-leaks-without-leaks:
@ -264,9 +257,9 @@ though neither Scrapy nor your project are leaking memory. This is due to a
(not so well) known problem of Python, which may not return released memory to
the operating system in some cases. For more information on this issue see:
* `Python Memory Management <http://www.evanjones.ca/python-memory.html>`_
* `Python Memory Management Part 2 <http://www.evanjones.ca/python-memory-part2.html>`_
* `Python Memory Management Part 3 <http://www.evanjones.ca/python-memory-part3.html>`_
* `Python Memory Management <https://www.evanjones.ca/python-memory.html>`_
* `Python Memory Management Part 2 <https://www.evanjones.ca/python-memory-part2.html>`_
* `Python Memory Management Part 3 <https://www.evanjones.ca/python-memory-part3.html>`_
The improvements proposed by Evan Jones, which are detailed in `this paper`_,
got merged in Python 2.5, but this only reduces the problem, it doesn't fix it
@ -280,7 +273,7 @@ completely. To quote the paper:
to move to a compacting garbage collector, which is able to move objects in
memory. This would require significant changes to the Python interpreter.*
.. _this paper: http://www.evanjones.ca/memoryallocator/
.. _this paper: https://www.evanjones.ca/memoryallocator/
To keep memory consumption reasonable you can split the job into several
smaller jobs or enable :ref:`persistent job queue <topics-jobs>`

81
docs/topics/link-extractors.rst
View File

@ -4,46 +4,33 @@
Link Extractors
===============
Link extractors are objects whose only purpose is to extract links from web
pages (:class:`scrapy.http.Response` objects) which will be eventually
followed.
A link extractor is an object that extracts links from responses.
There is ``scrapy.linkextractors import LinkExtractor`` available
in Scrapy, but you can create your own custom Link Extractors to suit your
needs by implementing a simple interface.
The only public method that every link extractor has is ``extract_links``,
which receives a :class:`~scrapy.http.Response` object and returns a list
of :class:`scrapy.link.Link` objects. Link extractors are meant to be
instantiated once and their ``extract_links`` method called several times
with different responses to extract links to follow.
Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider`
class (available in Scrapy), through a set of rules, but you can also use it in
your spiders, even if you don't subclass from
:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to
extract links.
The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
.. _topics-link-extractors-ref:
Built-in link extractors reference
==================================
Link extractor reference
========================
.. module:: scrapy.linkextractors
:synopsis: Link extractors classes
Link extractors classes bundled with Scrapy are provided in the
:mod:`scrapy.linkextractors` module.
The default link extractor is ``LinkExtractor``, which is the same as
:class:`~.LxmlLinkExtractor`::
The link extractor class is
:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
can also be imported as ``scrapy.linkextractors.LinkExtractor``::
from scrapy.linkextractors import LinkExtractor
There used to be other link extractor classes in previous Scrapy versions,
but they are deprecated now.
LxmlLinkExtractor
-----------------
@ -51,7 +38,7 @@ LxmlLinkExtractor
:synopsis: lxml's HTMLParser-based link extractors
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
LxmlLinkExtractor is the recommended link extractor with handy filtering
options. It is implemented using lxml's robust HTMLParser.
@ -62,7 +49,7 @@ LxmlLinkExtractor
:type allow: a regular expression (or list of)
:param deny: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be excluded (ie. not
that the (absolute) urls must match in order to be excluded (i.e. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type deny: a regular expression (or list of)
@ -77,9 +64,13 @@ LxmlLinkExtractor
:param deny_extensions: a single value or list of strings containing
extensions that should be ignored when extracting links.
If not given, it will default to the
``IGNORED_EXTENSIONS`` list defined in the
`scrapy.linkextractors`_ package.
If not given, it will default to
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
.. versionchanged:: 2.0
:data:`~scrapy.linkextractors.IGNORED_EXTENSIONS` now includes
``7z``, ``7zip``, ``apk``, ``bz2``, ``cdr``, ``dmg``, ``ico``,
``iso``, ``tar``, ``tar.gz``, ``webm``, and ``xz``.
:type deny_extensions: list
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
@ -93,6 +84,12 @@ LxmlLinkExtractor
Has the same behaviour as ``restrict_xpaths``.
:type restrict_css: str or list
:param restrict_text: a single regular expression (or list of regular expressions)
that the link's text must match in order to be extracted. If not
given (or empty), it will match all links. If a list of regular expressions is
given, the link will be extracted if it matches at least one.
:type restrict_text: a regular expression (or list of)
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
:type tags: str or list
@ -103,7 +100,12 @@ LxmlLinkExtractor
:type attrs: list
:param canonicalize: canonicalize each extracted url (using
w3lib.url.canonicalize_url). Defaults to ``True``.
w3lib.url.canonicalize_url). Defaults to ``False``.
Note that canonicalize_url is meant for duplicate checking;
it can change the URL visible at server side, so the response can be
different for requests with canonicalized and raw URLs. If you're
using LinkExtractor to follow links it is more robust to
keep the default ``canonicalize=False``.
:type canonicalize: boolean
:param unique: whether duplicate filtering should be applied to extracted
@ -132,4 +134,15 @@ LxmlLinkExtractor
:type process_value: callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
elements, etc., so LinkExtractor strips space chars by default.
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
.. automethod:: extract_links
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

162
docs/topics/loaders.rst
View File

@ -26,7 +26,7 @@ Using Item Loaders to populate items
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
which case an Item is automatically instantiated in the Item Loader constructor
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
the same item field; the Item Loader will know how to "join" those values later
using a proper processing function.
.. note:: Collected data is internally stored as lists,
allowing to add several values to the same field.
If an ``item`` argument is passed when creating a loader,
each of the item's values will be stored as-is if it's already
an iterable, or wrapped with a list if it's a single value.
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
chapter <topics-items>`::
@ -128,9 +134,12 @@ So what happens is:
It's worth noticing that processors are just callable objects, which are called
with the data to be parsed, and return a parsed value. So you can use any
function as input or output processor. The only requirement is that they must
accept one (and only one) positional argument, which will be an iterator.
accept one (and only one) positional argument, which will be an iterable.
.. note:: Both input and output processors must receive an iterator as their
.. versionchanged:: 2.0
Processors no longer need to be methods.
.. note:: Both input and output processors must receive an iterable as their
first argument. The output of those functions can be anything. The result of
input processors will be appended to an internal list (in the Loader)
containing the collected values (for that field). The result of the output
@ -144,6 +153,7 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors
<topics-loaders-available-processors>` built-in for convenience.
Declaring Item Loaders
======================
@ -199,14 +209,12 @@ metadata. Here is an example::
output_processor=TakeFirst(),
)
::
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
The precedence order, for both input and output processors, is as follows:
@ -250,7 +258,7 @@ There are several ways to modify Item Loader context values:
loader.context['unit'] = 'cm'
2. On Item Loader instantiation (the keyword arguments of Item Loader
constructor are stored in the Item Loader context)::
``__init__`` method are stored in the Item Loader context)::
loader = ItemLoader(product, unit='cm')
@ -271,7 +279,7 @@ ItemLoader objects
given, one is instantiated automatically using the class in
:attr:`default_item_class`.
When instantiated with a `selector` or a `response` parameters
When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
data from web pages using :ref:`selectors <topics-selectors>`.
@ -307,11 +315,11 @@ ItemLoader objects
applied before processors
:type re: str or compiled regex
Examples::
Examples:
>>> from scrapy.loader.processors import TakeFirst
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
>>> from scrapy.loader.processors import TakeFirst
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
.. method:: add_value(field_name, value, \*processors, \**kwargs)
@ -470,6 +478,8 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
This is mostly used as a property so when attempting to override this
value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
@ -479,7 +489,7 @@ ItemLoader objects
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
the constructor.
the ``__init__`` method.
.. attribute:: default_input_processor
@ -494,15 +504,15 @@ ItemLoader objects
.. attribute:: default_selector_class
The class used to construct the :attr:`selector` of this
:class:`ItemLoader`, if only a response is given in the constructor.
If a selector is given in the constructor this attribute is ignored.
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
If a selector is given in the ``__init__`` method this attribute is ignored.
This attribute is sometimes overridden in subclasses.
.. attribute:: selector
The :class:`~scrapy.selector.Selector` object to extract data from.
It's either the selector given in the constructor or one created from
the response given in the constructor using the
It's either the selector given in the ``__init__`` method or one created from
the response given in the ``__init__`` method using the
:attr:`default_selector_class`. This attribute is meant to be
read-only.
@ -518,8 +528,8 @@ a footer of a page that looks something like:
Example::
<footer>
<a class="social" href="http://facebook.com/whatever">Like Us</a>
<a class="social" href="http://twitter.com/whatever">Follow Us</a>
<a class="social" href="https://facebook.com/whatever">Like Us</a>
<a class="social" href="https://twitter.com/whatever">Follow Us</a>
<a class="email" href="mailto:whatever@example.com">Email Us</a>
</footer>
@ -627,46 +637,46 @@ Here is a list of all built-in processors:
.. class:: Identity
The simplest processor, which doesn't do anything. It returns the original
values unchanged. It doesn't receive any constructor arguments, nor does it
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
Example::
Example:
>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['one', 'two', 'three'])
['one', 'two', 'three']
>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['one', 'two', 'three'])
['one', 'two', 'three']
.. class:: TakeFirst
Returns the first non-null/non-empty value from the values received,
so it's typically used as an output processor to single-valued fields.
It doesn't receive any constructor arguments, nor does it accept Loader contexts.
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
Example::
Example:
>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'one', 'two', 'three'])
'one'
>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'one', 'two', 'three'])
'one'
.. class:: Join(separator=u' ')
Returns the values joined with the separator given in the constructor, which
Returns the values joined with the separator given in the ``__init__`` method, which
defaults to ``u' '``. It doesn't accept Loader contexts.
When using the default separator, this processor is equivalent to the
function: ``u' '.join``
Examples::
Examples:
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
u'one two three'
>>> proc = Join('<br>')
>>> proc(['one', 'two', 'three'])
u'one<br>two<br>three'
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
'one two three'
>>> proc = Join('<br>')
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
.. class:: Compose(\*functions, \**default_loader_context)
@ -679,18 +689,18 @@ Here is a list of all built-in processors:
By default, stop process on ``None`` value. This behaviour can be changed by
passing keyword argument ``stop_on_none=False``.
Example::
Example:
>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['hello', 'world'])
'HELLO'
>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['hello', 'world'])
'HELLO'
Each function can optionally receive a ``loader_context`` parameter. For
those which do, this processor will pass the currently active :ref:`Loader
context <topics-loaders-context>` through that parameter.
The keyword arguments passed in the constructor are used as the default
The keyword arguments passed in the ``__init__`` method are used as the default
Loader context values passed to each function call. However, the final
Loader context values passed to functions are overridden with the currently
active Loader context accessible through the :meth:`ItemLoader.context`
@ -723,41 +733,41 @@ Here is a list of all built-in processors:
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
<topics-selectors>`, which returns a list of unicode strings.
The example below should clarify how it works::
The example below should clarify how it works:
>>> def filter_world(x):
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
>>> proc = MapCompose(filter_world, unicode.upper)
>>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
[u'HELLO, u'THIS', u'IS', u'SCRAPY']
>>> def filter_world(x):
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
>>> proc = MapCompose(filter_world, str.upper)
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
constructor keyword arguments are used as default context values. See
``__init__`` method keyword arguments are used as default context values. See
:class:`Compose` processor for more info.
.. class:: SelectJmes(json_path)
Queries the value using the json path provided to the constructor and returns the output.
Queries the value using the json path provided to the ``__init__`` method and returns the output.
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
Example::
Example:
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
>>> proc({'foo': 'bar'})
'bar'
>>> proc({'foo': {'bar': 'baz'}})
{'bar': 'baz'}
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
>>> proc({'foo': 'bar'})
'bar'
>>> proc({'foo': {'bar': 'baz'}})
{'bar': 'baz'}
Working with Json::
Working with Json:
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
u'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
[u'bar']
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
['bar']

95
docs/topics/logging.rst
View File

@ -10,7 +10,7 @@ Logging
about the new logging system.
Scrapy uses `Python's builtin logging system
<https://docs.python.org/2/library/logging.html>`_ for event logging. We'll
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
provide some simple examples to get you started, but for more advanced
use-cases it's strongly suggested to read thoroughly its documentation.
@ -27,7 +27,7 @@ Scrapy from scripts as described in :ref:`run-from-script`.
Log levels
==========
Python's builtin logging defines 5 different levels to indicate severity on a
Python's builtin logging defines 5 different levels to indicate the severity of a
given log message. Here are the standard ones, listed in decreasing order:
1. ``logging.CRITICAL`` - for critical errors (highest severity)
@ -47,20 +47,20 @@ level::
There are shortcuts for issuing log messages on any of the standard 5 levels,
and there's also a general ``logging.log`` method which takes a given level as
argument. If you need so, last example could be rewrote as::
argument. If needed, the last example could be rewritten as::
import logging
logging.log(logging.WARNING, "This is a warning")
On top of that, you can create different "loggers" to encapsulate messages (For
example, a common practice it's to create different loggers for every module).
On top of that, you can create different "loggers" to encapsulate messages. (For
example, a common practice is to create different loggers for every module).
These loggers can be configured independently, and they allow hierarchical
constructions.
Last examples use the root logger behind the scenes, which is a top level
The previous examples use the root logger behind the scenes, which is a top level
logger where all messages are propagated to (unless otherwise specified). Using
``logging`` helpers is merely a shortcut for getting the root logger
explicitly, so this is also an equivalent of last snippets::
explicitly, so this is also an equivalent of the last snippets::
import logging
logger = logging.getLogger()
@ -95,14 +95,14 @@ Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
instance, that can be accessed and used like this::
instance, which can be accessed and used like this::
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://scrapinghub.com']
start_urls = ['https://scrapinghub.com']
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
@ -118,7 +118,7 @@ Python logger you want. For example::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://scrapinghub.com']
start_urls = ['https://scrapinghub.com']
def parse(self, response):
logger.info('Parse function called on %s', response.url)
@ -150,6 +150,7 @@ These settings can be used to configure the logging:
* :setting:`LOG_FORMAT`
* :setting:`LOG_DATEFORMAT`
* :setting:`LOG_STDOUT`
* :setting:`LOG_SHORT_NAMES`
The first couple of settings define a destination for log messages. If
:setting:`LOG_FILE` is set, messages sent through the root logger will be
@ -170,6 +171,10 @@ listed in `logging's logrecord attributes docs
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
respectively.
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
Scrapy component responsible for that log output.
Command-line options
--------------------
@ -188,6 +193,55 @@ to override some of the Scrapy settings regarding logging.
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
Further documentation on available handlers
.. _custom-log-formats:
Custom Log Formats
------------------
A custom log format can be set for different actions by extending
:class:`~scrapy.logformatter.LogFormatter` class and making
:setting:`LOG_FORMATTER` point to your new class.
.. autoclass:: scrapy.logformatter.LogFormatter
:members:
Advanced customization
----------------------
Because Scrapy uses stdlib logging module, you can customize logging using
all features of stdlib logging.
For example, let's say you're scraping a website which returns many
HTTP 404 and 500 responses, and you want to hide all messages like this::
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
is not handled or not allowed
The first thing to note is a logger name - it is in brackets:
``[scrapy.spidermiddlewares.httperror]``. If you get just ``[scrapy]`` then
:setting:`LOG_SHORT_NAMES` is likely set to True; set it to False and re-run
the crawl.
Next, we can see that the message has INFO level. To hide it
we should set logging level for ``scrapy.spidermiddlewares.httperror``
higher than INFO; next level after INFO is WARNING. It could be done
e.g. in the spider's ``__init__`` method::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('scrapy.spidermiddlewares.httperror')
logger.setLevel(logging.WARNING)
super().__init__(*args, **kwargs)
If you run this spider again then INFO messages from
``scrapy.spidermiddlewares.httperror`` logger will be gone.
scrapy.utils.log module
=======================
@ -196,22 +250,23 @@ scrapy.utils.log module
.. autofunction:: configure_logging
``configure_logging`` is automatically called when using Scrapy commands,
but needs to be called explicitly when running custom scripts. In that
case, its usage is not required but it's recommended.
``configure_logging`` is automatically called when using Scrapy commands
or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly
when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
In that case, its usage is not required but it's recommended.
If you plan on configuring the handlers yourself is still recommended you
call this function, passing `install_root_handler=False`. Bear in mind
there won't be any log output set by default in that case.
Another option when running custom scripts is to manually configure the logging.
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
To get you started on manually configuring logging's output, you can use
`logging.basicConfig()`_ to set a basic root handler. This is an example
on how to redirect ``INFO`` or higher messages to a file::
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
so it is recommended to only use `logging.basicConfig()`_ together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
import logging
from scrapy.utils.log import configure_logging
configure_logging(install_root_handler=False)
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',

174
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket)
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:
@ -96,7 +97,6 @@ For Files Pipeline, use::
ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
.. note::
You can also use both the Files and Images Pipeline at the same time.
@ -116,11 +116,6 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
Supported Storage
=================
File system is currently the only officially supported storage, but there is
also support for storing files in `Amazon S3`_.
.. _Amazon S3: https://aws.amazon.com/s3/
File system storage
-------------------
@ -130,7 +125,7 @@ For example, the following image URL::
http://www.example.com/image.jpg
Whose `SHA1 hash` is::
Whose ``SHA1 hash`` is::
3afec3b4765f8f0a07b78f98c07b83f013567a0a
@ -146,6 +141,29 @@ Where:
* ``full`` is a sub-directory to separate full images from thumbnails (if
used). For more info see :ref:`topics-images-thumbnails`.
.. _media-pipeline-ftp:
FTP server storage
------------------
.. versionadded:: 2.0
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
Scrapy will automatically upload the files to the server.
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` should be written in one of the
following forms::
ftp://username:password@address:port/path
ftp://address:port/path
If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
:setting:`FTP_PASSWORD` settings respectively.
FTP supports two different connection modes: active or passive. Scrapy uses
the passive connection mode by default. To use the active connection mode instead,
set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
Amazon S3 storage
-----------------
@ -169,7 +187,53 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _Minio: https://github.com/minio/minio
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Google Cloud Storage
---------------------
.. setting:: GCS_PROJECT_ID
.. setting:: FILES_STORE_GCS_ACL
.. setting:: IMAGES_STORE_GCS_ACL
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
IMAGES_STORE = 'gs://bucket/images/'
GCS_PROJECT_ID = 'project_id'
For information about authentication, see this `documentation`_.
.. _documentation: https://cloud.google.com/docs/authentication/production
You can modify the Access Control List (ACL) policy used for the stored files,
which is defined by the :setting:`FILES_STORE_GCS_ACL` and
:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
To make the files publicly available use the ``publicRead``
policy::
IMAGES_STORE_GCS_ACL = 'publicRead'
For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
Usage example
=============
@ -262,7 +326,7 @@ images.
.. setting:: IMAGES_THUMBS
In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
where the keys are the thumbnail names and the values are their dimensions.
For example::
@ -320,6 +384,18 @@ all be dropped because at least one dimension is shorter than the constraint.
By default, there are no size constraints, so all images are processed.
Allowing redirections
---------------------
.. setting:: MEDIA_ALLOW_REDIRECTS
By default media pipelines ignore redirects, i.e. an HTTP redirection
to a media file URL request will mean the media download is considered failed.
To handle media redirections, set this setting to ``True``::
MEDIA_ALLOW_REDIRECTS = True
.. _topics-media-pipeline-override:
Extending the Media Pipelines
@ -332,6 +408,36 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
.. method:: file_path(self, request, response=None, info=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
You can override this method to customize the download path of each file.
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@ -351,8 +457,9 @@ See here the methods that you can override in your custom Files Pipeline:
* ``success`` is a boolean which is ``True`` if the image was downloaded
successfully or ``False`` if it failed for some reason
* ``file_info_or_error`` is a dict containing the following keys (if success
is ``True``) or a `Twisted Failure`_ if there was a problem.
* ``file_info_or_error`` is a dict containing the following keys (if
success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
there was a problem.
* ``url`` - the url where the file was downloaded from. This is the url of
the request returned from the :meth:`~get_media_requests`
@ -415,6 +522,36 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
.. method:: file_path(self, request, response=None, info=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
You can override this method to customize the download path of each file.
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
@ -434,10 +571,12 @@ See here the methods that you can override in your custom Images Pipeline:
By default, the :meth:`item_completed` method returns the item.
.. _media-pipeline-example:
Custom Images pipeline example
==============================
Here is a full example of the Images Pipeline whose methods are examplified
Here is a full example of the Images Pipeline whose methods are exemplified
above::
import scrapy
@ -457,5 +596,12 @@ above::
item['image_paths'] = image_paths
return item
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
To enable your custom media pipeline component you must add its class import path to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
ITEM_PIPELINES = {
'myproject.pipelines.MyImagesPipeline': 300
}
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5

24
docs/topics/practices.rst
View File

@ -34,15 +34,16 @@ Here's an example showing how to run a single spider with it.
# Your spider definition
...
process = CrawlerProcess({
'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
process = CrawlerProcess(settings={
'FEED_FORMAT': 'json',
'FEED_URI': 'items.json'
})
process.crawl(MySpider)
process.start() # the script will block here until the crawling is finished
Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
acquainted with its usage details.
Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
documentation to get acquainted with its usage details.
If you are inside a Scrapy project there are some additional helpers you can
use to import those components within the project. You can automatically import
@ -80,7 +81,7 @@ returned by the :meth:`CrawlerRunner.crawl
<scrapy.crawler.CrawlerRunner.crawl>` method.
Here's an example of its usage, along with a callback to manually stop the
reactor after `MySpider` has finished running.
reactor after ``MySpider`` has finished running.
::
@ -100,7 +101,7 @@ reactor after `MySpider` has finished running.
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
.. seealso:: `Twisted Reactor Overview`_.
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
.. _run-multiple-spiders:
@ -238,7 +239,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
directly
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
@ -248,10 +249,9 @@ If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
.. _Tor project: https://www.torproject.org/
.. _commercial support: http://scrapy.org/support/
.. _ProxyMesh: http://proxymesh.com/
.. _commercial support: https://scrapy.org/support/
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
.. _Crawlera: http://scrapinghub.com/crawlera
.. _scrapoxy: http://scrapoxy.io/
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/

263
docs/topics/request-response.rst
View File

@ -24,13 +24,15 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
Request objects
===============
.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback])
.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
a :class:`Response`.
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
:type url: string
:param callback: the function that will be called with the response of this
@ -50,7 +52,7 @@ Request objects
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
``str`` using the `encoding` passed (which defaults to ``utf-8``). If
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
@ -80,18 +82,24 @@ Request objects
attributes of the cookie. This is only useful if the cookies are saved
for later requests.
.. reqmeta:: dont_merge_cookies
When some site returns cookies (in a response) those are stored in the
cookies for that domain and will be sent again in future requests. That's
the typical behaviour of any regular web browser. However, if, for some
reason, you want to avoid merging with existing cookies you can instruct
Scrapy to do so by setting the ``dont_merge_cookies`` key to True in the
:attr:`Request.meta`.
cookies for that domain and will be sent again in future requests.
That's the typical behaviour of any regular web browser.
Example of request without merging cookies::
To create a request that does not send stored cookies and does not
store received cookies, set the ``dont_merge_cookies`` key to ``True``
in :attr:`request.meta <scrapy.http.Request.meta>`.
request_with_cookies = Request(url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'},
meta={'dont_merge_cookies': True})
Example of a request that sends manually-defined cookies and ignores
cookie storage::
Request(
url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'},
meta={'dont_merge_cookies': True},
)
For more info see :ref:`cookies-mw`.
:type cookies: dict or list
@ -115,17 +123,27 @@ Request objects
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
as first parameter.
with 404 HTTP errors and such. It receives a
:exc:`~twisted.python.failure.Failure` as first parameter.
For more information,
see :ref:`topics-request-response-ref-errbacks` below.
.. versionchanged:: 2.0
The *callback* parameter is no longer required when the *errback*
parameter is specified.
:type errback: callable
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
:param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
:type cb_kwargs: dict
.. attribute:: Request.url
A string containing the URL of this request. Keep in mind that this
attribute contains the escaped URL, so it can differ from the URL passed in
the constructor.
the ``__init__`` method.
This attribute is read-only. To change the URL of a Request use
:meth:`replace`.
@ -160,6 +178,17 @@ Request objects
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
.. attribute:: Request.cb_kwargs
A dictionary that contains arbitrary metadata for this request. Its contents
will be passed to the Request's callback as keyword arguments. It is empty
for new Requests, which means by default callbacks only get a :class:`Response`
object as argument.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.cb_kwargs`` attribute.
.. _shallow copied: https://docs.python.org/2/library/copy.html
.. method:: Request.copy()
@ -167,14 +196,16 @@ Request objects
Return a new Request which is a copy of this Request. See also:
:ref:`topics-request-response-ref-request-callback-arguments`.
.. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
Return a Request object with the same members, except for those members
given new values by whichever keyword arguments are specified. The
attribute :attr:`Request.meta` is copied by default (unless a new value
is given in the ``meta`` argument). See also
:attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
copied by default (unless new values are given as arguments). See also
:ref:`topics-request-response-ref-request-callback-arguments`.
.. automethod:: from_curl
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@ -195,25 +226,31 @@ Example::
self.logger.info("Visited %s", response.url)
In some cases you may be interested in passing arguments to those callback
functions so you can receive the arguments later, in the second callback. You
can use the :attr:`Request.meta` attribute for that.
functions so you can receive the arguments later, in the second callback.
The following example shows how to achieve this by using the
:attr:`Request.cb_kwargs` attribute:
Here's an example of how to pass an item using this mechanism, to populate
different fields from different pages::
::
def parse_page1(self, response):
item = MyItem()
item['main_url'] = response.url
request = scrapy.Request("http://www.example.com/some_page.html",
callback=self.parse_page2)
request.meta['item'] = item
return request
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url))
request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback
yield request
def parse_page2(self, response):
item = response.meta['item']
item['other_url'] = response.url
return item
def parse_page2(self, response, main_url, foo):
yield dict(
main_url=main_url,
other_url=response.url,
foo=foo,
)
.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
Prior to that, using :attr:`Request.meta` was recommended for passing
information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
became the preferred way for handling user information, leaving :attr:`Request.meta`
for communication with components like middlewares and extensions.
.. _topics-request-response-ref-errbacks:
@ -223,8 +260,8 @@ Using errbacks to catch exceptions in request processing
The errback of a request is a function that will be called when an exception
is raise while processing it.
It receives a `Twisted Failure`_ instance as first parameter and can be
used to track connection establishment timeouts, DNS errors etc.
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
@ -291,16 +328,22 @@ Those are:
* :reqmeta:`dont_retry`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`handle_httpstatus_all`
* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`download_timeout`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_latency`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`proxy`
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* :reqmeta:`referrer_policy`
* :reqmeta:`max_retry_times`
.. reqmeta:: bindaddress
@ -327,6 +370,23 @@ started, i.e. HTTP message sent over the network. This meta key only becomes
available when the response has been downloaded. While most other meta keys are
used to control Scrapy behavior, this one is supposed to be read-only.
.. reqmeta:: download_fail_on_dataloss
download_fail_on_dataloss
-------------------------
Whether or not to fail on broken responses. See:
:setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
.. reqmeta:: max_retry_times
max_retry_times
---------------
The meta key is used set retry times per request. When initialized, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. _topics-request-response-ref-request-subclasses:
Request subclasses
@ -342,11 +402,11 @@ The FormRequest class extends the base :class:`Request` with functionality for
dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form
fields with form data from :class:`Response` objects.
.. _lxml.html forms: http://lxml.de/lxmlhtml.html#forms
.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
.. class:: FormRequest(url, [formdata, ...])
The :class:`FormRequest` class adds a new argument to the constructor. The
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@ -358,7 +418,7 @@ fields with form data from :class:`Response` objects.
The :class:`FormRequest` objects support the following class method in
addition to the standard :class:`Request` methods:
.. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
.. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
Returns a new :class:`FormRequest` object with its form field values
pre-populated with those found in the HTML ``<form>`` element contained
@ -376,6 +436,10 @@ fields with form data from :class:`Response` objects.
control clicked (instead of disabling it) you can also use the
``clickdata`` argument.
.. caution:: Using this method with select elements which have leading
or trailing whitespace in the option values will not work due to a
`bug in lxml`_, which should be fixed in lxml 3.8 and above.
:param response: the response containing a HTML form which will be used
to pre-populate the form fields
:type response: :class:`Response` object
@ -383,6 +447,9 @@ fields with form data from :class:`Response` objects.
:param formname: if given, the form with name attribute set to this value will be used.
:type formname: string
:param formid: if given, the form with id attribute set to this value will be used.
:type formid: string
:param formxpath: if given, the first form that matches the xpath will be used.
:type formxpath: string
@ -395,7 +462,9 @@ fields with form data from :class:`Response` objects.
:param formdata: fields to override in the form data. If a field was
already present in the response ``<form>`` element, its value is
overridden by the one passed in this parameter.
overridden by the one passed in this parameter. If a value passed in
this parameter is ``None``, the field will not be included in the
request, even if it was present in the response ``<form>`` element.
:type formdata: dict
:param clickdata: attributes to lookup the control clicked. If it's not
@ -410,7 +479,7 @@ fields with form data from :class:`Response` objects.
:type dont_click: boolean
The other parameters of this class method are passed directly to the
:class:`FormRequest` constructor.
:class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
@ -421,6 +490,9 @@ fields with form data from :class:`Response` objects.
.. versionadded:: 1.1.0
The ``formcss`` parameter.
.. versionadded:: 1.1.0
The ``formid`` parameter.
Request usage examples
----------------------
@ -450,6 +522,11 @@ method for this job. Here's an example spider which uses it::
import scrapy
def authentication_failed(response):
# TODO: Check the contents of the response and return True if it failed
# or False if it succeeded.
pass
class LoginSpider(scrapy.Spider):
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
@ -462,18 +539,55 @@ method for this job. Here's an example spider which uses it::
)
def after_login(self, response):
# check login succeed before going on
if "authentication failed" in response.body:
if authentication_failed(response):
self.logger.error("Login failed")
return
# continue scraping with authenticated session...
JsonRequest
-----------
The JsonRequest class extends the base :class:`Request` class with functionality for
dealing with JSON requests.
.. class:: JsonRequest(url, [... data, dumps_kwargs])
The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json``
and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
if :attr:`Request.body` argument is provided this parameter will be ignored.
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
set to ``'POST'`` automatically.
:type data: JSON serializable object
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
data into JSON format.
:type dumps_kwargs: dict
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
JsonRequest usage example
-------------------------
Sending a JSON POST request with a JSON payload::
data = {
'name1': 'value1',
'name2': 'value2',
}
yield JsonRequest(url='http://www.example.com/post/action', data=data)
Response objects
================
.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
.. autoclass:: Response
A :class:`Response` object represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
@ -488,11 +602,11 @@ Response objects
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
:param body: the response body. It must be str, not unicode, unless you're
using a encoding-aware :ref:`Response subclass
<topics-request-response-ref-response-subclasses>`, such as
:class:`TextResponse`.
:type body: str
:param body: the response body. To access the decoded text as str you can use
``response.text`` from an encoding-aware
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
such as :class:`TextResponse`.
:type body: bytes
:param flags: is a list containing the initial values for the
:attr:`Response.flags` attribute. If given, the list will be shallow
@ -501,7 +615,10 @@ Response objects
:param request: the initial value of the :attr:`Response.request` attribute.
This represents the :class:`Request` that generated this response.
:type request: :class:`Request` object
:type request: scrapy.http.Request
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
.. attribute:: Response.url
@ -556,7 +673,7 @@ Response objects
.. attribute:: Response.meta
A shortcut to the :attr:`Request.meta` attribute of the
:attr:`Response.request` object (ie. ``self.request.meta``).
:attr:`Response.request` object (i.e. ``self.request.meta``).
Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta`
attribute is propagated along redirects and retries, so you will get
@ -564,13 +681,34 @@ Response objects
.. seealso:: :attr:`Request.meta` attribute
.. attribute:: Response.cb_kwargs
.. versionadded:: 2.0
A shortcut to the :attr:`Request.cb_kwargs` attribute of the
:attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
Unlike the :attr:`Response.request` attribute, the
:attr:`Response.cb_kwargs` attribute is propagated along redirects and
retries, so you will get the original :attr:`Request.cb_kwargs` sent
from your spider.
.. seealso:: :attr:`Request.cb_kwargs` attribute
.. attribute:: Response.flags
A list that contains flags for this response. Flags are labels used for
tagging Responses. For example: `'cached'`, `'redirected`', etc. And
tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
they're shown on the string representation of the Response (`__str__`
method) which is used by the engine for logging.
.. attribute:: Response.certificate
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
Only populated for ``https`` responses, ``None`` otherwise.
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@ -591,6 +729,11 @@ Response objects
urlparse.urljoin(response.url, url)
.. automethod:: Response.follow
.. automethod:: Response.follow_all
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
.. _topics-request-response-ref-response-subclasses:
@ -610,7 +753,7 @@ TextResponse objects
:class:`Response` class, which is meant to be used only for binary data,
such as images, sounds or any media file.
:class:`TextResponse` objects support a new constructor argument, in
:class:`TextResponse` objects support a new ``__init__`` method argument, in
addition to the base :class:`Response` objects. The remaining functionality
is the same as for the :class:`Response` class and is not documented here.
@ -636,7 +779,7 @@ TextResponse objects
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
(typically `ascii`) instead of the response encoding.
(typically ``ascii``) instead of the response encoding.
.. attribute:: TextResponse.encoding
@ -644,10 +787,10 @@ TextResponse objects
A string with the encoding of this response. The encoding is resolved by
trying the following mechanisms, in order:
1. the encoding passed in the constructor `encoding` argument
1. the encoding passed in the ``__init__`` method ``encoding`` argument
2. the encoding declared in the Content-Type HTTP header. If this
encoding is not valid (ie. unknown), it is ignored and the next
encoding is not valid (i.e. unknown), it is ignored and the next
resolution mechanism is tried.
3. the encoding declared in the response body. The TextResponse class
@ -677,10 +820,14 @@ TextResponse objects
response.css('p')
.. automethod:: TextResponse.follow
.. automethod:: TextResponse.follow_all
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
kept for backwards compatibility; please prefer ``response.text``.
kept for backward compatibility; please prefer ``response.text``.
HtmlResponse objects
@ -692,7 +839,7 @@ HtmlResponse objects
which adds encoding auto-discovering support by looking into the HTML `meta
http-equiv`_ attribute. See :attr:`TextResponse.encoding`.
.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp
.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp
XmlResponse objects
-------------------
@ -703,4 +850,4 @@ XmlResponse objects
adds encoding auto-discovering support by looking into the XML declaration
line. See :attr:`TextResponse.encoding`.
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241

2
docs/topics/scrapyd.rst
View File

@ -10,4 +10,4 @@ Scrapyd has been moved into a separate project.
Its documentation is now hosted at:
http://scrapyd.readthedocs.org/en/latest/
https://scrapyd.readthedocs.io/en/latest/

1117
docs/topics/selectors.rst
View File

File diff suppressed because it is too large Load Diff

1
docs/topics/settings.rst
View File

@ -546,7 +546,6 @@ Default::
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
}

111
docs/topics/shell.rst
View File

@ -31,7 +31,7 @@ for more info.
Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
is unavailable.
Through scrapy's settings you can configure it to use any one of
Through Scrapy's settings you can configure it to use any one of
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
[settings]
shell = bpython
.. _IPython: http://ipython.org/
.. _IPython installation guide: http://ipython.org/install.html
.. _bpython: http://www.bpython-interpreter.org/
.. _IPython: https://ipython.org/
.. _IPython installation guide: https://ipython.org/install.html
.. _bpython: https://bpython-interpreter.org/
Launch the shell
================
@ -97,8 +97,12 @@ Available Shortcuts
* ``shelp()`` - print a help with the list of available objects and shortcuts
* ``fetch(request_or_url)`` - fetch a new response from the given request or
URL and update all related objects accordingly.
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
URL and update all related objects accordingly. You can optionaly ask for
HTTP 3xx redirections to not be followed by passing ``redirect=False``
* ``fetch(request)`` - fetch a new response from the given request and
update all related objects accordingly.
* ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
@ -138,7 +142,7 @@ Example of shell session
========================
Here's an example of a typical shell session where we start by scraping the
http://scrapy.org page, and then proceed to scrape the https://reddit.com
https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/
page. Finally, we modify the (Reddit) request method to POST and re-fetch it
getting an error. We end the session by typing Ctrl-D (in Unix systems) or
Ctrl-Z in Windows.
@ -150,55 +154,70 @@ shell works.
First, we launch the shell::
scrapy shell 'http://scrapy.org' --nolog
scrapy shell 'https://scrapy.org' --nolog
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
list of available objects and useful shortcuts (you'll notice that these lines
all start with the ``[s]`` prefix)::
[s] Available Scrapy objects:
[s] crawler <scrapy.crawler.Crawler object at 0x1e16b50>
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
[s] crawler <scrapy.crawler.Crawler object at 0x7f07395dd690>
[s] item {}
[s] request <GET http://scrapy.org>
[s] response <200 http://scrapy.org>
[s] settings <scrapy.settings.Settings object at 0x2bfd650>
[s] spider <Spider 'default' at 0x20c6f50>
[s] request <GET https://scrapy.org>
[s] response <200 https://scrapy.org/>
[s] settings <scrapy.settings.Settings object at 0x7f07395dd710>
[s] spider <DefaultSpider 'default' at 0x7f0735891690>
[s] Useful shortcuts:
[s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed)
[s] fetch(req) Fetch a scrapy.Request and update local objects
[s] shelp() Shell help (print this help)
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
[s] view(response) View response in a browser
>>>
After that, we can start playing with the objects::
>>> response.xpath('//title/text()').extract_first()
u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
After that, we can start playing with the objects:
>>> fetch("http://reddit.com")
[s] Available Scrapy objects:
[s] crawler <scrapy.crawler.Crawler object at 0x7fb3ed9c9c90>
[s] item {}
[s] request <GET http://reddit.com>
[s] response <200 https://www.reddit.com/>
[s] settings <scrapy.settings.Settings object at 0x7fb3ed9c9c10>
[s] spider <DefaultSpider 'default' at 0x7fb3ecdd3390>
[s] Useful shortcuts:
[s] shelp() Shell help (print this help)
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
[s] view(response) View response in a browser
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> response.xpath('//title/text()').extract()
[u'reddit: the front page of the internet']
>>> fetch("https://old.reddit.com/")
>>> request = request.replace(method="POST")
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'
>>> fetch(request)
[s] Available Scrapy objects:
[s] crawler <scrapy.crawler.Crawler object at 0x1e16b50>
...
>>> request = request.replace(method="POST")
>>> fetch(request)
>>> response.status
404
>>> from pprint import pprint
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
>>>
.. _topics-shell-inspect-response:
@ -234,8 +253,8 @@ Here's an example of how you would call it from your spider::
When you run the spider, you will get something similar to this::
2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.com> (referer: None)
2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.org> (referer: None)
[s] Available Scrapy objects:
[s] crawler <scrapy.crawler.Crawler object at 0x1e16b50>
...
@ -243,22 +262,22 @@ When you run the spider, you will get something similar to this::
>>> response.url
'http://example.org'
Then, you can check if the extraction code is working::
Then, you can check if the extraction code is working:
>>> response.xpath('//h1[@class="fn"]')
[]
>>> response.xpath('//h1[@class="fn"]')
[]
Nope, it doesn't. So you can open the response in your web browser and see if
it's the response you were expecting::
it's the response you were expecting:
>>> view(response)
True
>>> view(response)
True
Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
crawling::
>>> ^D
2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://example.net> (referer: None)
...
Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is

83
docs/topics/signals.rst
View File

@ -46,14 +46,15 @@ Here is a simple example showing how you can catch signals and perform some acti
def parse(self, response):
pass
.. _signal-deferred:
Deferred signal handlers
========================
Some signals support returning `Twisted deferreds`_ from their handlers, see
the :ref:`topics-signals-ref` below to know which ones.
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, see the :ref:`topics-signals-ref` below to know
which ones.
.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
.. _topics-signals-ref:
@ -135,6 +136,29 @@ item_dropped
to be dropped
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
item_error
------------
.. signal:: item_error
.. function:: item_error(item, response, spider, failure)
Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
spider_closed
-------------
@ -189,26 +213,32 @@ spider_idle
the engine starts closing the spider. After the spider has finished
closing, the :signal:`spider_closed` signal is sent.
You can, for example, schedule some requests in your :signal:`spider_idle`
handler to prevent the spider from being closed.
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
This signal does not support returning deferreds from their handlers.
:param spider: the spider which has gone idle
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: Scheduling some requests in your :signal:`spider_idle` handler does
**not** guarantee that it can prevent the spider from being closed,
although it sometimes can. That's because the spider may still remain idle
if all the scheduled requests are rejected by the scheduler (e.g. filtered
due to duplication).
spider_error
------------
.. signal:: spider_error
.. function:: spider_error(failure, response, spider)
Sent when a spider callback generates an error (ie. raises an exception).
Sent when a spider callback generates an error (i.e. raises an exception).
This signal does not support returning deferreds from their handlers.
:param failure: the exception raised as a Twisted `Failure`_ object
:type failure: `Failure`_ object
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
@ -250,6 +280,41 @@ request_dropped
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
request_reached_downloader
---------------------------
.. signal:: request_reached_downloader
.. function:: request_reached_downloader(request, spider)
Sent when a :class:`~scrapy.http.Request` reached downloader.
The signal does not support returning deferreds from their handlers.
:param request: the request that reached downloader
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
request_left_downloader
-----------------------
.. signal:: request_left_downloader
.. function:: request_left_downloader(request, spider)
.. versionadded:: 2.0
Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
failure.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the downloader
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
response_received
-----------------
@ -288,5 +353,3 @@ response_downloaded
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html

134
docs/topics/spider-middleware.rst
View File

@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied.
If you want to disable a builtin middleware (the ones defined in
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
value. For example, if you want to disable the off-site middleware::
SPIDER_MIDDLEWARES = {
@ -54,11 +54,17 @@ value. For example, if you want to disable the off-site middleware::
Finally, keep in mind that some middlewares may need to be enabled through a
particular setting. See each middleware documentation for more info.
.. _custom-spider-middleware:
Writing your own spider middleware
==================================
Each middleware component is a Python class that defines one or more of the
following methods:
Each spider middleware is a Python class that defines one or more of the
methods defined below.
The main entry point is the ``from_crawler`` class method, which receives a
:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
object gives you access, for example, to the :ref:`settings <topics-settings>`.
.. module:: scrapy.spidermiddlewares
@ -78,7 +84,8 @@ following methods:
If it raises an exception, Scrapy won't bother calling any other spider
middleware :meth:`process_spider_input` and will call the request
errback. The output of the errback is chained back in the other
errback if there is one, otherwise it will start the :meth:`process_spider_exception`
chain. The output of the errback is chained back in the other
direction for :meth:`process_spider_output` to process it, or
:meth:`process_spider_exception` if it raised an exception.
@ -95,7 +102,7 @@ following methods:
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
objects.
:param response: the response which generated this output from the
@ -112,11 +119,11 @@ following methods:
.. method:: process_spider_exception(response, exception, spider)
This method is called when when a spider or :meth:`process_spider_input`
method (from other spider middleware) raises an exception.
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.http.Response`, dict or
iterable of :class:`~scrapy.http.Request`, dict or
:class:`~scrapy.item.Item` objects.
If it returns ``None``, Scrapy will continue processing this exception,
@ -125,7 +132,8 @@ following methods:
exception reaches the engine (where it's logged and discarded).
If it returns an iterable the :meth:`process_spider_output` pipeline
kicks in, and no other :meth:`process_spider_exception` will be called.
kicks in, starting from the next spider middleware, and no other
:meth:`process_spider_exception` will be called.
:param response: the response being processed when the exception was
raised
@ -164,6 +172,17 @@ following methods:
:param spider: the spider to whom the start requests belong
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: from_crawler(cls, crawler)
If present, this classmethod is called to create a middleware instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the middleware. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for middleware to
access them and hook its functionality into Scrapy.
:param crawler: crawler that uses this middleware
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
@ -188,16 +207,21 @@ DepthMiddleware
.. class:: DepthMiddleware
DepthMiddleware is a scrape middleware used for tracking the depth of each
Request inside the site being scraped. It can be used to limit the maximum
depth to scrape or things like that.
DepthMiddleware is used for tracking the depth of each Request inside the
site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
there is no value previously set (usually just the first Request) and
incrementing it by 1 otherwise.
It can be used to limit the maximum depth to scrape, control Request
priority based on their depth, and things like that.
The :class:`DepthMiddleware` can be configured through the following
settings (see the settings documentation for more info):
* :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
crawl for any site. If zero, no limit will be imposed.
* :setting:`DEPTH_STATS` - Whether to collect depth stats.
* :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
requests for each depth.
* :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
their depth.
@ -328,6 +352,90 @@ Default: ``True``
Whether to enable referer middleware.
.. setting:: REFERRER_POLICY
REFERRER_POLICY
^^^^^^^^^^^^^^^
.. versionadded:: 1.4
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy
`Referrer Policy`_ to apply when populating Request "Referer" header.
.. note::
You can also set the Referrer Policy per request,
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
with the same acceptable values as for the ``REFERRER_POLICY`` setting.
Acceptable values for REFERRER_POLICY
*************************************
- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
subclass — a custom policy or one of the built-in ones (see classes below),
- or one of the standard W3C-defined string values,
- or the special ``"scrapy-default"``.
======================================= ========================================================================
String value Class name (as a string)
======================================= ========================================================================
``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy`
`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy`
`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy`
`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy`
`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy`
`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy`
`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy`
`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy`
`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`
======================================= ========================================================================
.. autoclass:: DefaultReferrerPolicy
.. warning::
Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
the W3C-recommended value for browsers — will send a non-empty
"Referer" header from any ``http(s)://`` to any ``https://`` URL,
even if the domain is different.
`"same-origin"`_ may be a better choice if you want to remove referrer
information for cross-domain requests.
.. autoclass:: NoReferrerPolicy
.. autoclass:: NoReferrerWhenDowngradePolicy
.. note::
"no-referrer-when-downgrade" policy is the W3C-recommended default,
and is used by major web browsers.
However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`).
.. autoclass:: SameOriginPolicy
.. autoclass:: OriginPolicy
.. autoclass:: StrictOriginPolicy
.. autoclass:: OriginWhenCrossOriginPolicy
.. autoclass:: StrictOriginWhenCrossOriginPolicy
.. autoclass:: UnsafeUrlPolicy
.. warning::
"unsafe-url" policy is NOT recommended.
.. _Referrer Policy: https://www.w3.org/TR/referrer-policy
.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
UrlLengthMiddleware
-------------------

186
docs/topics/spiders.rst
View File

@ -72,8 +72,6 @@ scrapy.Spider
spider that crawls ``mywebsite.com`` would often be called
``mywebsite``.
.. note:: In Python 2 this must be ASCII only.
.. attribute:: allowed_domains
An optional list of strings containing domains that this spider is
@ -81,11 +79,14 @@ scrapy.Spider
specified in this list (or their subdomains) won't be followed if
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
Let's say your target url is ``https://www.example.com/1.html``,
then add ``'example.com'`` to the list.
.. attribute:: start_urls
A list of URLs where the spider will begin to crawl from, when no
particular URLs are specified. So, the first pages downloaded will be those
listed here. The subsequent URLs will be generated successively from data
listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
contained in the start URLs.
.. attribute:: custom_settings
@ -126,7 +127,7 @@ scrapy.Spider
You probably won't need to override this directly because the default
implementation acts as a proxy to the :meth:`__init__` method, calling
it with the given arguments `args` and named arguments `kwargs`.
it with the given arguments ``args`` and named arguments ``kwargs``.
Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
attributes in the new instance so they can be accessed later inside the
@ -144,16 +145,12 @@ scrapy.Spider
.. method:: start_requests()
This method must return an iterable with the first Requests to crawl for
this spider.
this spider. It is called by Scrapy when the spider is opened for
scraping. Scrapy calls it only once, so it is safe to implement
:meth:`start_requests` as a generator.
This is the method called by Scrapy when the spider is opened for
scraping when no particular URLs are specified. If particular URLs are
specified, the :meth:`make_requests_from_url` is used instead to create
the Requests. This method is also called only once from Scrapy, so it's
safe to implement it as a generator.
The default implementation uses :meth:`make_requests_from_url` to
generate Requests for each url in :attr:`start_urls`.
The default implementation generates ``Request(url, dont_filter=True)``
for each url in :attr:`start_urls`.
If you want to change the Requests used to start scraping a domain, this is
the method to override. For example, if you need to start by logging in using
@ -172,18 +169,6 @@ scrapy.Spider
# each of them, with another callback
pass
.. method:: make_requests_from_url(url)
A method that receives a URL and returns a :class:`~scrapy.http.Request`
object (or a list of :class:`~scrapy.http.Request` objects) to scrape. This
method is used to construct the initial requests in the
:meth:`start_requests` method, and is typically used to convert urls to
requests.
Unless overridden, this method returns Requests with the :meth:`parse`
method as their callback function, and with dont_filter parameter enabled
(see :class:`~scrapy.http.Request` class for more info).
.. method:: parse(response)
This is the default callback used by Scrapy to process downloaded
@ -203,7 +188,7 @@ scrapy.Spider
.. method:: log(message, [level, component])
Wrapper that sends a log message through the Spider's :attr:`logger`,
kept for backwards compatibility. For more information see
kept for backward compatibility. For more information see
:ref:`topics-logging-from-spiders`.
.. method:: closed(reason)
@ -242,11 +227,11 @@ Return multiple Requests and items from a single callback::
]
def parse(self, response):
for h3 in response.xpath('//h3').extract():
for h3 in response.xpath('//h3').getall():
yield {"title": h3}
for url in response.xpath('//a/@href').extract():
yield scrapy.Request(url, callback=self.parse)
for href in response.xpath('//a/@href').getall():
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :ref:`topics-items`::
@ -264,11 +249,11 @@ to give data more structure you can use :ref:`topics-items`::
yield scrapy.Request('http://www.example.com/3.html', self.parse)
def parse(self, response):
for h3 in response.xpath('//h3').extract():
for h3 in response.xpath('//h3').getall():
yield MyItem(title=h3)
for url in response.xpath('//a/@href').extract():
yield scrapy.Request(url, callback=self.parse)
for href in response.xpath('//a/@href').getall():
yield scrapy.Request(response.urljoin(href), self.parse)
.. _spiderargs:
@ -297,6 +282,37 @@ Spiders can access arguments in their `__init__` methods::
self.start_urls = ['http://www.example.com/categories/%s' % category]
# ...
The default `__init__` method will take any spider arguments
and copy them to the spider as attributes.
The above example can also be written as follows::
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
def start_requests(self):
yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
If you were to set the ``start_urls`` attribute from the command line,
you would have to parse it on your own into a list
using something like
`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
and then set it as an attribute.
Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)
resulting in each character being seen as a separate url.
A valid use case is to set the http auth credentials
used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
or the user agent
used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`::
scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot
Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
See `Scrapyd documentation`_.
@ -356,16 +372,23 @@ CrawlSpider
Crawling rules
~~~~~~~~~~~~~~
.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
.. autoclass:: Rule
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
defines how links will be extracted from each crawled page.
defines how links will be extracted from each crawled page. Each produced link will
be used to generate a :class:`~scrapy.http.Request` object, which will contain the
link's text in its ``meta`` dictionary (under the ``link_text`` key).
If omitted, a default link extractor created with no arguments will be used,
resulting in all links being extracted.
``callback`` is a callable or a string (in which case a method from the spider
object with that name will be used) to be called for each link extracted with
the specified link_extractor. This callback receives a response as its first
argument and must return a list containing :class:`~scrapy.item.Item` and/or
:class:`~scrapy.http.Request` objects (or any subclass of them).
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
as its first argument and must return either a single instance or an iterable of
:class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
callback, since the :class:`CrawlSpider` uses the ``parse`` method
@ -384,10 +407,21 @@ Crawling rules
of links extracted from each response using the specified ``link_extractor``.
This is mainly used for filtering purposes.
``process_request`` is a callable, or a string (in which case a method from
the spider object with that name will be used) which will be called with
every request extracted by this rule, and must return a request or None (to
filter out the request).
``process_request`` is a callable (or a string, in which case a method from
the spider object with that name will be used) which will be called for every
:class:`~scrapy.http.Request` extracted by this rule. This callable should
take said request as first argument and the :class:`~scrapy.http.Response`
from which the request originated as second argument. It must return a
``Request`` object or ``None`` (to filter out the request).
``errback`` is a callable or a string (in which case a method from the spider
object with that name will be used) to be called if any exception is
raised while processing a request generated by the rule.
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
.. versionadded:: 2.0
The *errback* parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
@ -416,8 +450,9 @@ Let's now take a look at an example CrawlSpider with rules::
self.logger.info('Hi, this is an item page! %s', response.url)
item = scrapy.Item()
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
return item
@ -527,12 +562,12 @@ These spiders are pretty easy to use, let's have a look at one example::
itertag = 'item'
def parse_node(self, response, node):
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
item = TestItem()
item['id'] = node.xpath('@id').extract()
item['name'] = node.xpath('name').extract()
item['description'] = node.xpath('description').extract()
item['id'] = node.xpath('@id').get()
item['name'] = node.xpath('name').get()
item['description'] = node.xpath('description').get()
return item
Basically what we did up there was to create a spider that downloads a feed from
@ -560,8 +595,7 @@ CSVFeedSpider
.. attribute:: headers
A list of the rows contained in the file CSV feed which will be used to
extract fields from it.
A list of the column names in the CSV file.
.. method:: parse_row(response, row)
@ -638,7 +672,7 @@ SitemapSpider
.. attribute:: sitemap_follow
A list of regexes of sitemap that should be followed. This is is only
A list of regexes of sitemap that should be followed. This is only
for sites that use `Sitemap index files`_ that point to other sitemap
files.
@ -663,6 +697,50 @@ SitemapSpider
Default is ``sitemap_alternate_links`` disabled.
.. method:: sitemap_filter(entries)
This is a filter function that could be overridden to select sitemap entries
based on their attributes.
For example::
<url>
<loc>http://example.com/</loc>
<lastmod>2005-01-01</lastmod>
</url>
We can define a ``sitemap_filter`` function to filter ``entries`` by date::
from datetime import datetime
from scrapy.spiders import SitemapSpider
class FilteredSitemapSpider(SitemapSpider):
name = 'filtered_sitemap_spider'
allowed_domains = ['example.com']
sitemap_urls = ['http://example.com/sitemap.xml']
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
if date_time.year >= 2005:
yield entry
This would retrieve only ``entries`` modified on 2005 and the following
years.
Entries are dict objects extracted from the sitemap document.
Usually, the key is the tag name and the value is the text inside it.
It's important to notice that:
- as the loc attribute is required, entries without this tag are discarded
- alternate links are stored in a list with the key ``alternate``
(see ``sitemap_alternate_links``)
- namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``
If you omit this method, all entries found in sitemaps will be
processed, observing other attributes and their settings.
SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
@ -734,8 +812,8 @@ Combine SitemapSpider with other sources of urls::
def parse_other(self, response):
pass # ... scrape other here ...
.. _Sitemaps: http://www.sitemaps.org
.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
.. _robots.txt: http://www.robotstxt.org/
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
.. _robots.txt: https://www.robotstxt.org/
.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
.. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/
.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

17
docs/topics/stats.rst
View File

@ -32,7 +32,7 @@ Common Stats Collector uses
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
class ExtensionThatAccessStats(object):
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats
@ -57,15 +57,15 @@ Set stat value only if lower than previous::
stats.min_value('min_free_memory_percent', value)
Get stat value::
Get stat value:
>>> stats.get_value('custom_count')
1
>>> stats.get_value('custom_count')
1
Get all stats::
Get all stats:
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================
@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
default Stats Collector used is the :class:`MemoryStatsCollector`.
.. module:: scrapy.statscollectors
:synopsis: Stats Collectors
.. currentmodule:: scrapy.statscollectors
MemoryStatsCollector
--------------------

57
docs/topics/telnetconsole.rst
View File

@ -1,12 +1,11 @@
.. currentmodule:: scrapy.extensions.telnet
.. _topics-telnetconsole:
==============
Telnet Console
==============
.. module:: scrapy.extensions.telnet
:synopsis: The Telnet Console
Scrapy comes with a built-in telnet console for inspecting and controlling a
Scrapy running process. The telnet console is just a regular python shell
running inside the Scrapy process, so you can do literally anything from it.
@ -16,6 +15,17 @@ The telnet console is a :ref:`built-in Scrapy extension
disable it if you want. For more information about the extension itself see
:ref:`topics-extensions-ref-telnetconsole`.
.. warning::
It is not secure to use telnet console via public networks, as telnet
doesn't provide any transport-layer security. Having username/password
authentication doesn't change that.
Intended usage is connecting to a running Scrapy spider locally
(spider process and telnet client are on the same machine)
or over a secure connection (VPN, SSH tunnel).
Please avoid using telnet console over insecure connections,
or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option.
.. highlight:: none
How to access the telnet console
@ -26,8 +36,26 @@ The telnet console listens in the TCP port defined in the
the console you need to type::
telnet localhost 6023
Trying localhost...
Connected to localhost.
Escape character is '^]'.
Username:
Password:
>>>
By default Username is ``scrapy`` and Password is autogenerated. The
autogenerated Password can be seen on Scrapy logs like the example below::
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
Default Username and Password can be overridden by the settings
:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
.. warning::
Username and password provide only a limited protection, as telnet
is not using secure transport - by default traffic is not encrypted
even if username and password are set.
You need the telnet program which comes installed by default in Windows, and
most Linux distros.
@ -160,3 +188,24 @@ Default: ``'127.0.0.1'``
The interface the telnet console should listen on
.. setting:: TELNETCONSOLE_USERNAME
TELNETCONSOLE_USERNAME
----------------------
Default: ``'scrapy'``
The username used for the telnet console
.. setting:: TELNETCONSOLE_PASSWORD
TELNETCONSOLE_PASSWORD
----------------------
Default: ``None``
The password used for the telnet console, default behaviour is to have it
autogenerated

39
docs/topics/ubuntu.rst
View File

@ -1,39 +0,0 @@
.. _topics-ubuntu:
===============
Ubuntu packages
===============
.. versionadded:: 0.10
`Scrapinghub`_ publishes apt-gettable packages which are generally fresher than
those in Ubuntu, and more stable too since they're continuously built from
`GitHub repo`_ (master & stable branches) and so they contain the latest bug
fixes.
.. caution:: These packages are currently not updated and may not work on
Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`.
To use the packages:
1. Import the GPG key used to sign Scrapy packages into APT keyring::
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command::
echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
3. Update package lists and install the scrapy package:
.. parsed-literal::
sudo apt-get update && sudo apt-get install scrapy
.. note:: Repeat step 3 if you are trying to upgrade Scrapy.
.. warning:: `python-scrapy` is a different package provided by official debian
repositories, it's very outdated and it isn't supported by Scrapy team.
.. _Scrapinghub: http://scrapinghub.com/
.. _GitHub repo: https://github.com/scrapy/scrapy

2
docs/utils/linkfix.py
View File

@ -20,7 +20,7 @@ _filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:

2
docs/versioning.rst
View File

@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
* *A* is the major version. This will rarely change and will signify very
large changes.
* *B* is the release number. This will include many changes including features
and things that possibly break backwards compatibility, although we strive to
and things that possibly break backward compatibility, although we strive to
keep theses cases at a minimum.
* *C* is the bugfix release number.

2
extras/coverage-report.sh
View File

@ -1,6 +1,6 @@
# Run tests, generate coverage report and open it on a browser
#
# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage
# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage
coverage run --branch $(which trial) --reporter=text tests
coverage html -i

1
extras/qps-bench-server.py
View File

@ -1,5 +1,4 @@
#!/usr/bin/env python
from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET

2
extras/qpsclient.py
View File

@ -41,7 +41,7 @@ class QPSSpider(Spider):
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)]
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
else:
urls = [url]

8
extras/scrapy.1
View File

@ -28,16 +28,16 @@ Query Scrapy settings
Print raw setting value
.TP
.I --getbool=SETTING
Print setting value, intepreted as a boolean
Print setting value, interpreted as a boolean
.TP
.I --getint=SETTING
Print setting value, intepreted as an integer
Print setting value, interpreted as an integer
.TP
.I --getfloat=SETTING
Print setting value, intepreted as an float
Print setting value, interpreted as a float
.TP
.I --getlist=SETTING
Print setting value, intepreted as an float
Print setting value, interpreted as a float
.TP
.I --init
Print initial setting value (before loading extensions and spiders)

226
extras/scrapy_zsh_completion
View File

@ -1,25 +1,213 @@
#compdef scrapy
# zsh completion for the Scrapy command-line tool
_scrapy() {
local curcontext="$curcontext" cmd spiders
local context state state_descr line
local ret=1
typeset -A opt_args
cmd=$words[2]
case "$cmd" in
crawl|edit|check)
spiders=$(scrapy list 2>/dev/null) || spiders=""
if [[ -n "$spiders" ]]; then
compadd `echo $spiders`
fi
;;
*)
if [[ CURRENT -eq 2 ]]; then
_arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
fi
;;
_arguments \
"(- 1 *)"{-h,--help}"[Help]" \
"1: :->command" \
"*:: :->args" && ret=0
case $state in
command)
_scrapy_cmds
;;
args)
case $words[1] in
(bench)
_scrapy_glb_opts
;;
(fetch)
local options=(
'--headers[print response HTTP headers instead of body]'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
(genspider)
local options=(
{'(--list)-l','(-l)--list'}'[List available templates]'
{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
'--force[If the spider already exists, overwrite it with the template]'
{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
'1:name:(NAME)'
'2:domain:_httpie_urls'
)
_scrapy_glb_opts $options
;;
(runspider)
local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py'
)
_scrapy_glb_opts $options
;;
(settings)
local options=(
'--get=[print raw setting value]:option:(SETTING)'
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
'--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
'--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
'--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
)
_scrapy_glb_opts $options
;;
(shell)
local options=(
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider=[use this spider]:spider:_scrapy_spiders'
'::file:_files -g \*.html'
'::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
(startproject)
local options=(
'1:name:(NAME)'
'2:dir:_dir_list'
)
_scrapy_glb_opts $options
;;
(version)
local options=(
{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
)
_scrapy_glb_opts $options
;;
(view)
local options=(
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
(check)
local options=(
{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
(crawl)
local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
(edit)
local options=(
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
(list)
_scrapy_glb_opts
;;
(parse)
local options=(
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
'--pipelines[process items through pipelines]'
"--nolinks[don't show links to follow (extracted requests)]"
"--noitems[don't show scraped items]"
'--nocolour[avoid using pygments to colorize the output]'
{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
esac
;;
esac
return ret
}
_scrapy
_scrapy_cmds() {
local -a commands project_commands
commands=(
'bench:Run quick benchmark test'
'fetch:Fetch a URL using the Scrapy downloader'
'genspider:Generate new spider using pre-defined templates'
'runspider:Run a self-contained spider (without creating a project)'
'settings:Get settings values'
'shell:Interactive scraping console'
'startproject:Create new project'
'version:Print Scrapy version'
'view:Open URL in browser, as seen by Scrapy'
)
project_commands=(
'check:Check spider contracts'
'crawl:Run a spider'
'edit:Edit spider'
'list:List available spiders'
'parse:Parse URL (using its spider) and print the results'
)
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
commands=(${commands[@]} ${project_commands[@]})
fi
_describe -t common-commands 'common commands' commands && ret=0
}
_scrapy_glb_opts() {
local -a options
options=(
'(- *)'{-h,--help}'[show this help message and exit]'
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
'--pidfile=[write process ID to FILE]:file:_files'
'--profile=[write python cProfile stats to FILE]:file:_files'
{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
'(-L --loglevel --logfile)--nolog[disable logging completely]'
'--pdb[enable pdb on failure]'
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
)
options=(${options[@]} "$@")
_arguments -A "-*" $options && ret=0
}
_httpie_urls() {
local ret=1
if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
local expl
compset -S '[^:/]*' && compstate[to_end]=''
_wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
else
_urls && ret=0
fi
return $ret
}
_scrapy_spiders() {
local ret=1
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
compadd -S '' $(scrapy list) && ret=0
else
compadd -S '' SPIDER && ret=0
fi
return $ret
}
_scrapy $@

247
pytest.ini
View File

@ -2,5 +2,250 @@
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
addopts = --doctest-modules --assert=plain
addopts =
--assert=plain
--doctest-modules
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
--ignore=docs/topics/dynamic-content.rst
--ignore=docs/topics/items.rst
--ignore=docs/topics/leaks.rst
--ignore=docs/topics/loaders.rst
--ignore=docs/topics/selectors.rst
--ignore=docs/topics/shell.rst
--ignore=docs/topics/stats.rst
--ignore=docs/topics/telnetconsole.rst
--ignore=docs/utils
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-ignore =
W503
# Files that are only meant to provide top-level imports are expected not
# to use any of their imports:
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
# Issues pending a review:
# extras
extras/qps-bench-server.py E501
extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127 E502
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128 W504
scrapy/core/spidermw.py E501 E731 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E241
scrapy/core/downloader/webclient.py E731 E501 E128 E126
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E501
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
scrapy/extensions/closespider.py E501 E128 E123
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/throttle.py E501
# scrapy/http
scrapy/http/common.py E501
scrapy/http/cookies.py E501
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501 E731
# scrapy/loader
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/__init__.py E501
scrapy/pipelines/files.py E116 E501 E266
scrapy/pipelines/images.py E265 E501
scrapy/pipelines/media.py E125 E501 E266
# scrapy/selector
scrapy/selector/__init__.py F403
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
scrapy/settings/default_settings.py E501 E114 E116
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
scrapy/spidermiddlewares/referer.py E501 E129 W504
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/asyncio.py E501
scrapy/utils/benchserver.py E501
scrapy/utils/conf.py E402 E501
scrapy/utils/datatypes.py E501
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/gz.py E501 W504
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501
scrapy/utils/log.py E128 E501
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501
scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E501
scrapy/utils/reqser.py E501
scrapy/utils/request.py E127 E501
scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
scrapy/utils/spider.py E501
scrapy/utils/ssl.py E501
scrapy/utils/test.py E501
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/resolver.py E501
scrapy/responsetypes.py E128 E501
scrapy/robotstxt.py E501
scrapy/shell.py E501
scrapy/signalmanager.py E501
scrapy/spiderloader.py F841 E501 E126
scrapy/squeues.py E128
scrapy/statscollectors.py E501
# tests
tests/__init__.py E402 E501
tests/mockserver.py E401 E501 E126 E123
tests/pipelines.py F841
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E128 E127
tests/test_downloadermiddleware_retry.py E501 E128 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E128 E265
tests/test_item.py E128 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E501 E731 E741 E128 E117 E241
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128 E126
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E731 E128 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E241
tests/test_utils_log.py E741
tests/test_utils_python.py E501 E731
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501

7
requirements-py3.txt
View File

@ -1,7 +0,0 @@
Twisted >= 15.5.0
lxml>=3.2.4
pyOpenSSL>=0.13.1
cssselect>=0.9
queuelib>=1.1.1
w3lib>=1.14.2
service_identity

Some files were not shown because too many files have changed in this diff Show More